From 7190b38b8d2569e8d7a9f65e93d08ff44308fc81 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Tue, 4 Oct 2022 09:36:16 +0200 Subject: [PATCH 01/44] [3.2.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index e053a23cb3a..ad16ceec43f 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 16, 'final', 0) +VERSION = (3, 2, 17, 'alpha', 0) __version__ = get_version(VERSION) From accdd0576d50b4696378ed1ab651c20affcae08b Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Tue, 4 Oct 2022 10:12:11 +0200 Subject: [PATCH 02/44] [3.2.x] Added CVE-2022-36359 to security archive. Backport of 93d4c9ea1de24eb391cb2b3561b6703fd46374df from main --- docs/releases/security.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/releases/security.txt b/docs/releases/security.txt index f039379e0ed..940f91fa0fb 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -36,6 +36,17 @@ Issues under Django's security process All security issues have been handled under versions of Django's security process. These are listed below. +October 4, 2022 - :cve:`2022-41323` +----------------------------------- + +Potential denial-of-service vulnerability in internationalized URLs. `Full +description +`__ + +* Django 4.1 :commit:`(patch) <9d656ea51d9ea7105c0c0785783ac29d426a7d25>` +* Django 4.0 :commit:`(patch) <23f0093125ac2e553da6c1b2f9988eb6a3dd2ea1>` +* Django 3.2 :commit:`(patch) <5b6b257fa7ec37ff27965358800c67e2dd11c924>` + August 3, 2022 - :cve:`2022-36359` ---------------------------------- From f6f0699d01f5840437bfd236c76c797943ef8edc Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Sat, 29 Oct 2022 12:34:22 +0100 Subject: [PATCH 03/44] [3.2.x] Removed obsolete doc reference to asyncio.iscoroutinefunction. Backport of 970f61fefb148284fb2af63b5cc844279254111a from main --- docs/topics/http/middleware.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/http/middleware.txt b/docs/topics/http/middleware.txt index aa89f79e0cf..d82a3f6ae89 100644 --- a/docs/topics/http/middleware.txt +++ b/docs/topics/http/middleware.txt @@ -314,7 +314,7 @@ If your middleware has both ``sync_capable = True`` and ``async_capable = True``, then Django will pass it the request without converting it. In this case, you can work out if your middleware will receive async requests by checking if the ``get_response`` object you are passed is a -coroutine function, using :py:func:`asyncio.iscoroutinefunction`. +coroutine function, using ``asyncio.iscoroutinefunction``. The ``django.utils.decorators`` module contains :func:`~django.utils.decorators.sync_only_middleware`, From b381ab4906cdff0ebb4e1c6c8e30996f24be6829 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Thu, 29 Dec 2022 06:07:51 +0100 Subject: [PATCH 04/44] [3.2.x] Disabled auto-created table of contents entries on Sphinx 5.2+. Auto-created table of contents entries for all domain objects (e.g. functions, classes, attributes, etc.) were added in Sphinx 5.2, see https://github.com/sphinx-doc/sphinx/issues/6316. An option to control new table of contents entries was added in Sphinx 5.2.3, see https://github.com/sphinx-doc/sphinx/pull/10886. Backport of 279967ec859a9a5240318cf29a077539b0e3139f from main --- docs/conf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index fabe91cc389..42d350052cb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -73,6 +73,10 @@ # The root toctree document. root_doc = "contents" +# Disable auto-created table of contents entries for all domain objects (e.g. +# functions, classes, attributes, etc.) in Sphinx 5.2+. +toc_object_entries = False + # General substitutions. project = 'Django' copyright = 'Django Software Foundation and contributors' From 238e8898ac0486f60ca3567d2459cda1d51912f3 Mon Sep 17 00:00:00 2001 From: Stephen <101494292+stephenatgithub@users.noreply.github.com> Date: Wed, 25 Jan 2023 13:09:57 +0800 Subject: [PATCH 05/44] [3.2.x] Corrected passenv value for tox 4.0.6+. Backport of 34b328814976a2e2f7907361a494202763649f3f from main --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6f3c09fddb1..3b74bb5a135 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ basepython = python3 [testenv] usedevelop = true # OBJC_DISABLE_INITIALIZE_FORK_SAFETY fixes hung tests for MacOS users. (#30806) -passenv = DJANGO_SETTINGS_MODULE PYTHONPATH HOME DISPLAY OBJC_DISABLE_INITIALIZE_FORK_SAFETY +passenv = DJANGO_SETTINGS_MODULE,PYTHONPATH,HOME,DISPLAY,OBJC_DISABLE_INITIALIZE_FORK_SAFETY setenv = PYTHONDONTWRITEBYTECODE=1 deps = From 4e31d3ea55315811db00f3f0d5c677e2344c2f5d Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 25 Jan 2023 11:57:04 +0100 Subject: [PATCH 06/44] [3.2.x] Added stub release notes for 3.2.17. Backport of 1df963ad2476726d63be132c0cee47e07b8250d7 from main --- docs/releases/3.2.17.txt | 9 +++++++++ docs/releases/index.txt | 1 + 2 files changed, 10 insertions(+) create mode 100644 docs/releases/3.2.17.txt diff --git a/docs/releases/3.2.17.txt b/docs/releases/3.2.17.txt new file mode 100644 index 00000000000..6d1118e70cc --- /dev/null +++ b/docs/releases/3.2.17.txt @@ -0,0 +1,9 @@ +=========================== +Django 3.2.17 release notes +=========================== + +*February 1, 2023* + +Django 3.2.17 fixes a security issue in 3.2.16. + +... diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 74370dd7479..212993d736d 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.2.17 3.2.16 3.2.15 3.2.14 From d21543182d2cb9947650ecc48c068d1bfb7d0311 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 25 Jan 2023 12:26:00 +0100 Subject: [PATCH 07/44] [3.2.x] Adjusted release notes for 3.2.17. Backport of d8e1442ce2c56282785dd806e5c1147975e8c857 from main --- docs/releases/3.2.17.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releases/3.2.17.txt b/docs/releases/3.2.17.txt index 6d1118e70cc..9eba24d72f1 100644 --- a/docs/releases/3.2.17.txt +++ b/docs/releases/3.2.17.txt @@ -4,6 +4,6 @@ Django 3.2.17 release notes *February 1, 2023* -Django 3.2.17 fixes a security issue in 3.2.16. +Django 3.2.17 fixes a security issue with severity "moderate" in 3.2.16. ... From 4c2b26174f044adc4a6461154385720479eaee55 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sun, 1 May 2022 21:44:04 -0400 Subject: [PATCH 08/44] [3.2.x] Removed 'tests' path prefix in a couple tests. Backport of 694cf458f16b8d340a3195244196980b2dec34fd from main. --- tests/i18n/tests.py | 2 +- tests/sessions_tests/tests.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/i18n/tests.py b/tests/i18n/tests.py index 7edceb1c95c..4c31c3bc664 100644 --- a/tests/i18n/tests.py +++ b/tests/i18n/tests.py @@ -1948,7 +1948,7 @@ def test_i18n_locale_paths(self): def test_i18n_app_dirs(self): mocked_sender = mock.MagicMock() - with self.settings(INSTALLED_APPS=['tests.i18n.sampleproject']): + with self.settings(INSTALLED_APPS=["i18n.sampleproject"]): watch_for_translation_changes(mocked_sender) project_dir = Path(__file__).parent / 'sampleproject' / 'locale' mocked_sender.watch_dir.assert_any_call(project_dir, '**/*.mo') diff --git a/tests/sessions_tests/tests.py b/tests/sessions_tests/tests.py index 73d2a13a9f3..ad5dbdb6ddf 100644 --- a/tests/sessions_tests/tests.py +++ b/tests/sessions_tests/tests.py @@ -915,9 +915,9 @@ def test_session_save_does_not_resurrect_session_logged_out_in_other_context(sel class ClearSessionsCommandTests(SimpleTestCase): def test_clearsessions_unsupported(self): msg = ( - "Session engine 'tests.sessions_tests.no_clear_expired' doesn't " + "Session engine 'sessions_tests.no_clear_expired' doesn't " "support clearing expired sessions." ) - with self.settings(SESSION_ENGINE='tests.sessions_tests.no_clear_expired'): + with self.settings(SESSION_ENGINE="sessions_tests.no_clear_expired"): with self.assertRaisesMessage(management.CommandError, msg): management.call_command('clearsessions') From 9da46345d83e5d9ecb60512efb2d2e0b2b02b974 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Thu, 9 Dec 2021 20:24:38 +0100 Subject: [PATCH 09/44] [3.2.x] Fixed inspectdb.tests.InspectDBTestCase.test_custom_fields() on SQLite 3.37+. Use FlexibleFieldLookupDict which is case-insensitive mapping because SQLite 3.37+ returns some data type names upper-cased e.g. TEXT. Backport of 974e3b8750fe96c16c9c0b115a72ee4a2171df34 from main --- tests/inspectdb/tests.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/inspectdb/tests.py b/tests/inspectdb/tests.py index a0c4aff2e5a..8ef757fc18e 100644 --- a/tests/inspectdb/tests.py +++ b/tests/inspectdb/tests.py @@ -302,18 +302,17 @@ def test_custom_fields(self): Introspection of columns with a custom field (#21090) """ out = StringIO() - orig_data_types_reverse = connection.introspection.data_types_reverse - try: - connection.introspection.data_types_reverse = { + with mock.patch( + 'django.db.connection.introspection.data_types_reverse.base_data_types_reverse', + { 'text': 'myfields.TextField', 'bigint': 'BigIntegerField', - } + }, + ): call_command('inspectdb', 'inspectdb_columntypes', stdout=out) output = out.getvalue() self.assertIn("text_field = myfields.TextField()", output) self.assertIn("big_int_field = models.BigIntegerField()", output) - finally: - connection.introspection.data_types_reverse = orig_data_types_reverse def test_introspection_errors(self): """ From c7e0151fdf33e1b11d488b6f67b94fdf3a30614a Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 25 Jan 2023 12:21:48 +0100 Subject: [PATCH 10/44] [3.2.x] Fixed CVE-2023-23969 -- Prevented DoS with pathological values for Accept-Language. The parsed values of Accept-Language headers are cached in order to avoid repetitive parsing. This leads to a potential denial-of-service vector via excessive memory usage if the raw value of Accept-Language headers is very large. Accept-Language headers are now limited to a maximum length in order to avoid this issue. --- django/utils/translation/trans_real.py | 32 +++++++++++++++++++++++++- docs/releases/3.2.17.txt | 10 +++++++- tests/i18n/tests.py | 12 ++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py index 8042f6fdc41..b262a5000a4 100644 --- a/django/utils/translation/trans_real.py +++ b/django/utils/translation/trans_real.py @@ -30,6 +30,11 @@ # magic gettext number to separate context from message CONTEXT_SEPARATOR = "\x04" +# Maximum number of characters that will be parsed from the Accept-Language +# header to prevent possible denial of service or memory exhaustion attacks. +# About 10x longer than the longest value shown on MDN’s Accept-Language page. +ACCEPT_LANGUAGE_HEADER_MAX_LENGTH = 500 + # Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9 # and RFC 3066, section 2.1 accept_language_re = _lazy_re_compile(r''' @@ -556,7 +561,7 @@ def get_language_from_request(request, check_path=False): @functools.lru_cache(maxsize=1000) -def parse_accept_lang_header(lang_string): +def _parse_accept_lang_header(lang_string): """ Parse the lang_string, which is the body of an HTTP Accept-Language header, and return a tuple of (lang, q-value), ordered by 'q' values. @@ -578,3 +583,28 @@ def parse_accept_lang_header(lang_string): result.append((lang, priority)) result.sort(key=lambda k: k[1], reverse=True) return tuple(result) + + +def parse_accept_lang_header(lang_string): + """ + Parse the value of the Accept-Language header up to a maximum length. + + The value of the header is truncated to a maximum length to avoid potential + denial of service and memory exhaustion attacks. Excessive memory could be + used if the raw value is very large as it would be cached due to the use of + functools.lru_cache() to avoid repetitive parsing of common header values. + """ + # If the header value doesn't exceed the maximum allowed length, parse it. + if len(lang_string) <= ACCEPT_LANGUAGE_HEADER_MAX_LENGTH: + return _parse_accept_lang_header(lang_string) + + # If there is at least one comma in the value, parse up to the last comma + # before the max length, skipping any truncated parts at the end of the + # header value. + index = lang_string.rfind(",", 0, ACCEPT_LANGUAGE_HEADER_MAX_LENGTH) + if index > 0: + return _parse_accept_lang_header(lang_string[:index]) + + # Don't attempt to parse if there is only one language-range value which is + # longer than the maximum allowed length and so truncated. + return () diff --git a/docs/releases/3.2.17.txt b/docs/releases/3.2.17.txt index 9eba24d72f1..fcc097c5ccd 100644 --- a/docs/releases/3.2.17.txt +++ b/docs/releases/3.2.17.txt @@ -6,4 +6,12 @@ Django 3.2.17 release notes Django 3.2.17 fixes a security issue with severity "moderate" in 3.2.16. -... +CVE-2023-23969: Potential denial-of-service via ``Accept-Language`` headers +=========================================================================== + +The parsed values of ``Accept-Language`` headers are cached in order to avoid +repetitive parsing. This leads to a potential denial-of-service vector via +excessive memory usage if large header values are sent. + +In order to avoid this vulnerability, the ``Accept-Language`` header is now +parsed up to a maximum length. diff --git a/tests/i18n/tests.py b/tests/i18n/tests.py index 4c31c3bc664..41ec63da99e 100644 --- a/tests/i18n/tests.py +++ b/tests/i18n/tests.py @@ -1352,6 +1352,14 @@ def test_parse_spec_http_header(self): ('de;q=0.', [('de', 0.0)]), ('en; q=1,', [('en', 1.0)]), ('en; q=1.0, * ; q=0.5', [('en', 1.0), ('*', 0.5)]), + ( + 'en' + '-x' * 20, + [('en-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x', 1.0)], + ), + ( + ', '.join(['en; q=1.0'] * 20), + [('en', 1.0)] * 20, + ), # Bad headers ('en-gb;q=1.0000', []), ('en;q=0.1234', []), @@ -1367,6 +1375,10 @@ def test_parse_spec_http_header(self): ('12-345', []), ('', []), ('en;q=1e0', []), + # Invalid as language-range value too long. + ('xxxxxxxx' + '-xxxxxxxx' * 500, []), + # Header value too long, only parse up to limit. + (', '.join(['en; q=1.0'] * 500), [('en', 1.0)] * 45), ] for value, expected in tests: with self.subTest(value=value): From aed1bb56d118937d5d6f3ec72f170779dd8c74cd Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 1 Feb 2023 09:58:36 +0100 Subject: [PATCH 11/44] [3.2.x] Bumped version for 3.2.17 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index ad16ceec43f..37b8060e425 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 17, 'alpha', 0) +VERSION = (3, 2, 17, 'final', 0) __version__ = get_version(VERSION) From 9bd8db3940f529aebafb348c7d6786f29a288916 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 1 Feb 2023 10:00:34 +0100 Subject: [PATCH 12/44] [3.2.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 37b8060e425..b3db8f8bcc7 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 17, 'final', 0) +VERSION = (3, 2, 18, 'alpha', 0) __version__ = get_version(VERSION) From c35a5788f4c17c580976458b0b04210a91133d20 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 1 Feb 2023 12:09:03 +0100 Subject: [PATCH 13/44] [3.2.x] Added CVE-2023-23969 to security archive. Backport of 36e3eef7d5a4c88671d20a561788679d0d9c334c from main --- docs/releases/security.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/releases/security.txt b/docs/releases/security.txt index 940f91fa0fb..0023fed03fd 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -36,6 +36,16 @@ Issues under Django's security process All security issues have been handled under versions of Django's security process. These are listed below. +February 1, 2023 - :cve:`2023-23969` +------------------------------------ + +Potential denial-of-service via ``Accept-Language`` headers. `Full description +`__ + +* Django 4.1 :commit:`(patch) <9d7bd5a56b1ce0576e8e07a8001373576d277942>` +* Django 4.0 :commit:`(patch) <4452642f193533e288a52c02efb5bbc766a68f95>` +* Django 3.2 :commit:`(patch) ` + October 4, 2022 - :cve:`2022-41323` ----------------------------------- From 932b5bd52d8d7e9255264fdbf425e322efac0b97 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Thu, 2 Feb 2023 11:54:09 +0100 Subject: [PATCH 14/44] [3.2.x] Added stub release notes for 3.2.18. Backport of 7e003428f96d616c1f77fed84882a95e63bc3644 from main --- docs/releases/3.2.18.txt | 9 +++++++++ docs/releases/index.txt | 1 + 2 files changed, 10 insertions(+) create mode 100644 docs/releases/3.2.18.txt diff --git a/docs/releases/3.2.18.txt b/docs/releases/3.2.18.txt new file mode 100644 index 00000000000..431d04c9897 --- /dev/null +++ b/docs/releases/3.2.18.txt @@ -0,0 +1,9 @@ +=========================== +Django 3.2.18 release notes +=========================== + +*February 14, 2023* + +Django 3.2.18 fixes a security issue with severity "moderate" in 3.2.17. + +... diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 212993d736d..de60213273f 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.2.18 3.2.17 3.2.16 3.2.15 From a665ed5179f5bbd3db95ce67286d0192eff041d8 Mon Sep 17 00:00:00 2001 From: Markus Holtermann Date: Tue, 13 Dec 2022 10:27:39 +0100 Subject: [PATCH 15/44] [3.2.x] Fixed CVE-2023-24580 -- Prevented DoS with too many uploaded files. Thanks to Jakob Ackermann for the report. --- django/conf/global_settings.py | 4 ++ django/core/exceptions.py | 9 +++ django/core/handlers/exception.py | 4 +- django/http/multipartparser.py | 62 +++++++++++++++++---- django/http/request.py | 6 +- docs/ref/exceptions.txt | 5 ++ docs/ref/settings.txt | 23 ++++++++ docs/releases/3.2.18.txt | 10 +++- tests/handlers/test_exception.py | 28 +++++++++- tests/requests/test_data_upload_settings.py | 51 ++++++++++++++++- 10 files changed, 184 insertions(+), 18 deletions(-) diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index cf9fae496e3..4a27887a8f0 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -303,6 +303,10 @@ def gettext_noop(s): # SuspiciousOperation (TooManyFieldsSent) is raised. DATA_UPLOAD_MAX_NUMBER_FIELDS = 1000 +# Maximum number of files encoded in a multipart upload that will be read +# before a SuspiciousOperation (TooManyFilesSent) is raised. +DATA_UPLOAD_MAX_NUMBER_FILES = 100 + # Directory in which upload streamed files will be temporarily saved. A value of # `None` will make Django use the operating system's default temporary directory # (i.e. "/tmp" on *nix systems). diff --git a/django/core/exceptions.py b/django/core/exceptions.py index 673d004d575..83161a58cd6 100644 --- a/django/core/exceptions.py +++ b/django/core/exceptions.py @@ -58,6 +58,15 @@ class TooManyFieldsSent(SuspiciousOperation): pass +class TooManyFilesSent(SuspiciousOperation): + """ + The number of fields in a GET or POST request exceeded + settings.DATA_UPLOAD_MAX_NUMBER_FILES. + """ + + pass + + class RequestDataTooBig(SuspiciousOperation): """ The size of the request (excluding any file uploads) exceeded diff --git a/django/core/handlers/exception.py b/django/core/handlers/exception.py index 3005a5eccb1..2ecc2a0fd69 100644 --- a/django/core/handlers/exception.py +++ b/django/core/handlers/exception.py @@ -9,7 +9,7 @@ from django.core import signals from django.core.exceptions import ( BadRequest, PermissionDenied, RequestDataTooBig, SuspiciousOperation, - TooManyFieldsSent, + TooManyFieldsSent, TooManyFilesSent, ) from django.http import Http404 from django.http.multipartparser import MultiPartParserError @@ -88,7 +88,7 @@ def response_for_exception(request, exc): exc_info=sys.exc_info(), ) elif isinstance(exc, SuspiciousOperation): - if isinstance(exc, (RequestDataTooBig, TooManyFieldsSent)): + if isinstance(exc, (RequestDataTooBig, TooManyFieldsSent, TooManyFilesSent)): # POST data can't be accessed again, otherwise the original # exception would be raised. request._mark_post_parse_error() diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index 35a54f4ca12..d8a304d4bab 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -14,6 +14,7 @@ from django.conf import settings from django.core.exceptions import ( RequestDataTooBig, SuspiciousMultipartForm, TooManyFieldsSent, + TooManyFilesSent, ) from django.core.files.uploadhandler import ( SkipFile, StopFutureHandlers, StopUpload, @@ -38,6 +39,7 @@ class InputStreamExhausted(Exception): RAW = "raw" FILE = "file" FIELD = "field" +FIELD_TYPES = frozenset([FIELD, RAW]) class MultiPartParser: @@ -102,6 +104,22 @@ def __init__(self, META, input_data, upload_handlers, encoding=None): self._upload_handlers = upload_handlers def parse(self): + # Call the actual parse routine and close all open files in case of + # errors. This is needed because if exceptions are thrown the + # MultiPartParser will not be garbage collected immediately and + # resources would be kept alive. This is only needed for errors because + # the Request object closes all uploaded files at the end of the + # request. + try: + return self._parse() + except Exception: + if hasattr(self, "_files"): + for _, files in self._files.lists(): + for fileobj in files: + fileobj.close() + raise + + def _parse(self): """ Parse the POST data and break it into a FILES MultiValueDict and a POST MultiValueDict. @@ -147,6 +165,8 @@ def parse(self): num_bytes_read = 0 # To count the number of keys in the request. num_post_keys = 0 + # To count the number of files in the request. + num_files = 0 # To limit the amount of data read from the request. read_size = None # Whether a file upload is finished. @@ -162,6 +182,20 @@ def parse(self): old_field_name = None uploaded_file = True + if ( + item_type in FIELD_TYPES and + settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None + ): + # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS. + num_post_keys += 1 + # 2 accounts for empty raw fields before and after the + # last boundary. + if settings.DATA_UPLOAD_MAX_NUMBER_FIELDS + 2 < num_post_keys: + raise TooManyFieldsSent( + "The number of GET/POST parameters exceeded " + "settings.DATA_UPLOAD_MAX_NUMBER_FIELDS." + ) + try: disposition = meta_data['content-disposition'][1] field_name = disposition['name'].strip() @@ -174,15 +208,6 @@ def parse(self): field_name = force_str(field_name, encoding, errors='replace') if item_type == FIELD: - # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS. - num_post_keys += 1 - if (settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None and - settings.DATA_UPLOAD_MAX_NUMBER_FIELDS < num_post_keys): - raise TooManyFieldsSent( - 'The number of GET/POST parameters exceeded ' - 'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.' - ) - # Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE. if settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None: read_size = settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read @@ -208,6 +233,16 @@ def parse(self): self._post.appendlist(field_name, force_str(data, encoding, errors='replace')) elif item_type == FILE: + # Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FILES. + num_files += 1 + if ( + settings.DATA_UPLOAD_MAX_NUMBER_FILES is not None and + num_files > settings.DATA_UPLOAD_MAX_NUMBER_FILES + ): + raise TooManyFilesSent( + "The number of files exceeded " + "settings.DATA_UPLOAD_MAX_NUMBER_FILES." + ) # This is a file, use the handler... file_name = disposition.get('filename') if file_name: @@ -276,8 +311,13 @@ def parse(self): # Handle file upload completions on next iteration. old_field_name = field_name else: - # If this is neither a FIELD or a FILE, just exhaust the stream. - exhaust(stream) + # If this is neither a FIELD nor a FILE, exhaust the field + # stream. Note: There could be an error here at some point, + # but there will be at least two RAW types (before and + # after the other boundaries). This branch is usually not + # reached at all, because a missing content-disposition + # header will skip the whole boundary. + exhaust(field_stream) except StopUpload as e: self._close_files() if not e.connection_reset: diff --git a/django/http/request.py b/django/http/request.py index 195341ec4b6..b6cd7a372f1 100644 --- a/django/http/request.py +++ b/django/http/request.py @@ -12,7 +12,9 @@ DisallowedHost, ImproperlyConfigured, RequestDataTooBig, TooManyFieldsSent, ) from django.core.files import uploadhandler -from django.http.multipartparser import MultiPartParser, MultiPartParserError +from django.http.multipartparser import ( + MultiPartParser, MultiPartParserError, TooManyFilesSent, +) from django.utils.datastructures import ( CaseInsensitiveMapping, ImmutableList, MultiValueDict, ) @@ -360,7 +362,7 @@ def _load_post_and_files(self): data = self try: self._post, self._files = self.parse_file_upload(self.META, data) - except MultiPartParserError: + except (MultiPartParserError, TooManyFilesSent): # An error occurred while parsing POST data. Since when # formatting the error the request handler might access # self.POST, set self._post and self._file to prevent diff --git a/docs/ref/exceptions.txt b/docs/ref/exceptions.txt index 2f5aa64b9d9..7d34025cd65 100644 --- a/docs/ref/exceptions.txt +++ b/docs/ref/exceptions.txt @@ -84,12 +84,17 @@ Django core exception classes are defined in ``django.core.exceptions``. * ``SuspiciousMultipartForm`` * ``SuspiciousSession`` * ``TooManyFieldsSent`` + * ``TooManyFilesSent`` If a ``SuspiciousOperation`` exception reaches the ASGI/WSGI handler level it is logged at the ``Error`` level and results in a :class:`~django.http.HttpResponseBadRequest`. See the :doc:`logging documentation ` for more information. +.. versionchanged:: 3.2.18 + + ``SuspiciousOperation`` is raised when too many files are submitted. + ``PermissionDenied`` -------------------- diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 9bfadbc89bd..9173009c94d 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1063,6 +1063,28 @@ could be used as a denial-of-service attack vector if left unchecked. Since web servers don't typically perform deep request inspection, it's not possible to perform a similar check at that level. +.. setting:: DATA_UPLOAD_MAX_NUMBER_FILES + +``DATA_UPLOAD_MAX_NUMBER_FILES`` +-------------------------------- + +.. versionadded:: 3.2.18 + +Default: ``100`` + +The maximum number of files that may be received via POST in a +``multipart/form-data`` encoded request before a +:exc:`~django.core.exceptions.SuspiciousOperation` (``TooManyFiles``) is +raised. You can set this to ``None`` to disable the check. Applications that +are expected to receive an unusually large number of file fields should tune +this setting. + +The number of accepted files is correlated to the amount of time and memory +needed to process the request. Large requests could be used as a +denial-of-service attack vector if left unchecked. Since web servers don't +typically perform deep request inspection, it's not possible to perform a +similar check at that level. + .. setting:: DATABASE_ROUTERS ``DATABASE_ROUTERS`` @@ -3671,6 +3693,7 @@ HTTP ---- * :setting:`DATA_UPLOAD_MAX_MEMORY_SIZE` * :setting:`DATA_UPLOAD_MAX_NUMBER_FIELDS` +* :setting:`DATA_UPLOAD_MAX_NUMBER_FILES` * :setting:`DEFAULT_CHARSET` * :setting:`DISALLOWED_USER_AGENTS` * :setting:`FORCE_SCRIPT_NAME` diff --git a/docs/releases/3.2.18.txt b/docs/releases/3.2.18.txt index 431d04c9897..46c0feb51e4 100644 --- a/docs/releases/3.2.18.txt +++ b/docs/releases/3.2.18.txt @@ -6,4 +6,12 @@ Django 3.2.18 release notes Django 3.2.18 fixes a security issue with severity "moderate" in 3.2.17. -... +CVE-2023-24580: Potential denial-of-service vulnerability in file uploads +========================================================================= + +Passing certain inputs to multipart forms could result in too many open files +or memory exhaustion, and provided a potential vector for a denial-of-service +attack. + +The number of files parts parsed is now limited via the new +:setting:`DATA_UPLOAD_MAX_NUMBER_FILES` setting. diff --git a/tests/handlers/test_exception.py b/tests/handlers/test_exception.py index 0c1e7639904..7de2edaeea3 100644 --- a/tests/handlers/test_exception.py +++ b/tests/handlers/test_exception.py @@ -1,6 +1,8 @@ from django.core.handlers.wsgi import WSGIHandler from django.test import SimpleTestCase, override_settings -from django.test.client import FakePayload +from django.test.client import ( + BOUNDARY, MULTIPART_CONTENT, FakePayload, encode_multipart, +) class ExceptionHandlerTests(SimpleTestCase): @@ -25,3 +27,27 @@ def test_data_upload_max_memory_size_exceeded(self): def test_data_upload_max_number_fields_exceeded(self): response = WSGIHandler()(self.get_suspicious_environ(), lambda *a, **k: None) self.assertEqual(response.status_code, 400) + + @override_settings(DATA_UPLOAD_MAX_NUMBER_FILES=2) + def test_data_upload_max_number_files_exceeded(self): + payload = FakePayload( + encode_multipart( + BOUNDARY, + { + "a.txt": "Hello World!", + "b.txt": "Hello Django!", + "c.txt": "Hello Python!", + }, + ) + ) + environ = { + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": MULTIPART_CONTENT, + "CONTENT_LENGTH": len(payload), + "wsgi.input": payload, + "SERVER_NAME": "test", + "SERVER_PORT": "8000", + } + + response = WSGIHandler()(environ, lambda *a, **k: None) + self.assertEqual(response.status_code, 400) diff --git a/tests/requests/test_data_upload_settings.py b/tests/requests/test_data_upload_settings.py index 44897cc9fa9..ded778b4228 100644 --- a/tests/requests/test_data_upload_settings.py +++ b/tests/requests/test_data_upload_settings.py @@ -1,11 +1,14 @@ from io import BytesIO -from django.core.exceptions import RequestDataTooBig, TooManyFieldsSent +from django.core.exceptions import ( + RequestDataTooBig, TooManyFieldsSent, TooManyFilesSent, +) from django.core.handlers.wsgi import WSGIRequest from django.test import SimpleTestCase from django.test.client import FakePayload TOO_MANY_FIELDS_MSG = 'The number of GET/POST parameters exceeded settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.' +TOO_MANY_FILES_MSG = 'The number of files exceeded settings.DATA_UPLOAD_MAX_NUMBER_FILES.' TOO_MUCH_DATA_MSG = 'Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.' @@ -166,6 +169,52 @@ def test_no_limit(self): self.request._load_post_and_files() +class DataUploadMaxNumberOfFilesMultipartPost(SimpleTestCase): + def setUp(self): + payload = FakePayload( + "\r\n".join( + [ + "--boundary", + ( + 'Content-Disposition: form-data; name="name1"; ' + 'filename="name1.txt"' + ), + "", + "value1", + "--boundary", + ( + 'Content-Disposition: form-data; name="name2"; ' + 'filename="name2.txt"' + ), + "", + "value2", + "--boundary--", + ] + ) + ) + self.request = WSGIRequest( + { + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": "multipart/form-data; boundary=boundary", + "CONTENT_LENGTH": len(payload), + "wsgi.input": payload, + } + ) + + def test_number_exceeded(self): + with self.settings(DATA_UPLOAD_MAX_NUMBER_FILES=1): + with self.assertRaisesMessage(TooManyFilesSent, TOO_MANY_FILES_MSG): + self.request._load_post_and_files() + + def test_number_not_exceeded(self): + with self.settings(DATA_UPLOAD_MAX_NUMBER_FILES=2): + self.request._load_post_and_files() + + def test_no_limit(self): + with self.settings(DATA_UPLOAD_MAX_NUMBER_FILES=None): + self.request._load_post_and_files() + + class DataUploadMaxNumberOfFieldsFormPost(SimpleTestCase): def setUp(self): payload = FakePayload("\r\n".join(['a=1&a=2&a=3', ''])) From 722e9f8a38f5b34f2423059a75f8a59bb8eb931a Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Tue, 14 Feb 2023 09:04:22 +0100 Subject: [PATCH 16/44] [3.2.x] Bumped version for 3.2.18 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index b3db8f8bcc7..28f109f9d8e 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 18, 'alpha', 0) +VERSION = (3, 2, 18, 'final', 0) __version__ = get_version(VERSION) From e34a2283f2b66cbc0239b9443544bacac0576ef1 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Tue, 14 Feb 2023 09:07:53 +0100 Subject: [PATCH 17/44] [3.2.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 28f109f9d8e..396186c816e 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 18, 'final', 0) +VERSION = (3, 2, 19, 'alpha', 0) __version__ = get_version(VERSION) From 963f24cff2820d2b2902da0a9218e93c3933e713 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Tue, 14 Feb 2023 09:52:30 +0100 Subject: [PATCH 18/44] [3.2.x] Added CVE-2023-24580 to security archive. Backport of ecafcaf634fcef93f9da8cb12795273dd1c3a576 from main --- docs/releases/security.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/releases/security.txt b/docs/releases/security.txt index 0023fed03fd..0a827387094 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -36,6 +36,16 @@ Issues under Django's security process All security issues have been handled under versions of Django's security process. These are listed below. +February 14, 2023 - :cve:`2023-24580` +------------------------------------- + +Potential denial-of-service vulnerability in file uploads. `Full description +`__ + +* Django 4.1 :commit:`(patch) <628b33a854a9c68ec8a0c51f382f304a0044ec92>` +* Django 4.0 :commit:`(patch) <83f1ea83e4553e211c1c5a0dfc197b66d4e50432>` +* Django 3.2 :commit:`(patch) ` + February 1, 2023 - :cve:`2023-23969` ------------------------------------ From a37e4d5d6ec5df97d81df0793b390d61212e645e Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Thu, 20 Apr 2023 09:01:59 +0200 Subject: [PATCH 19/44] [3.2.x] Added stub release notes for 3.2.19. Backport of 18a7f2c711529f8e43c36190a5e2479f13899749 from main --- docs/releases/3.2.19.txt | 9 +++++++++ docs/releases/index.txt | 1 + 2 files changed, 10 insertions(+) create mode 100644 docs/releases/3.2.19.txt diff --git a/docs/releases/3.2.19.txt b/docs/releases/3.2.19.txt new file mode 100644 index 00000000000..c5817e689c5 --- /dev/null +++ b/docs/releases/3.2.19.txt @@ -0,0 +1,9 @@ +=========================== +Django 3.2.19 release notes +=========================== + +*May 3, 2023* + +Django 3.2.19 fixes a security issue with severity "low" in 3.2.18. + +... diff --git a/docs/releases/index.txt b/docs/releases/index.txt index de60213273f..0b045b730aa 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.2.19 3.2.18 3.2.17 3.2.16 From 007e46d815063d598e0d3db78bfb371100e5c61c Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 26 Apr 2023 09:30:14 +0200 Subject: [PATCH 20/44] [3.2.x] Added missing backticks in docs/releases/1.7.txt. --- docs/releases/1.7.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index 6403ae0b517..814ff6db230 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -1196,8 +1196,8 @@ manager will *not* be reset. This was necessary to resolve an inconsistency in the way routing information cascaded over joins. See :ticket:`13724` for more details. -pytz may be required --------------------- +``pytz`` may be required +------------------------ If your project handles datetimes before 1970 or after 2037 and Django raises a :exc:`ValueError` when encountering them, you will have to install pytz_. You From eed53d0011622e70b936e203005f0e6f4ac48965 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Thu, 13 Apr 2023 10:10:56 +0200 Subject: [PATCH 21/44] [3.2.x] Fixed CVE-2023-31047, Fixed #31710 -- Prevented potential bypass of validation when uploading multiple files using one form field. Thanks Moataz Al-Sharida and nawaik for reports. Co-authored-by: Shai Berger Co-authored-by: nessita <124304+nessita@users.noreply.github.com> --- django/forms/widgets.py | 26 ++++++- docs/releases/3.2.19.txt | 16 ++++- docs/topics/http/file-uploads.txt | 65 ++++++++++++++++-- .../forms_tests/field_tests/test_filefield.py | 68 ++++++++++++++++++- .../widget_tests/test_clearablefileinput.py | 5 ++ .../widget_tests/test_fileinput.py | 44 ++++++++++++ 6 files changed, 215 insertions(+), 9 deletions(-) diff --git a/django/forms/widgets.py b/django/forms/widgets.py index 1b1c1439cb4..8ef82552a19 100644 --- a/django/forms/widgets.py +++ b/django/forms/widgets.py @@ -378,16 +378,40 @@ def format_value(self, value): class FileInput(Input): input_type = 'file' + allow_multiple_selected = False needs_multipart_form = True template_name = 'django/forms/widgets/file.html' + def __init__(self, attrs=None): + if ( + attrs is not None and + not self.allow_multiple_selected and + attrs.get("multiple", False) + ): + raise ValueError( + "%s doesn't support uploading multiple files." + % self.__class__.__qualname__ + ) + if self.allow_multiple_selected: + if attrs is None: + attrs = {"multiple": True} + else: + attrs.setdefault("multiple", True) + super().__init__(attrs) + def format_value(self, value): """File input never renders a value.""" return def value_from_datadict(self, data, files, name): "File widgets take data from FILES, not POST" - return files.get(name) + getter = files.get + if self.allow_multiple_selected: + try: + getter = files.getlist + except AttributeError: + pass + return getter(name) def value_omitted_from_data(self, data, files, name): return name not in files diff --git a/docs/releases/3.2.19.txt b/docs/releases/3.2.19.txt index c5817e689c5..9f9eb3f45c7 100644 --- a/docs/releases/3.2.19.txt +++ b/docs/releases/3.2.19.txt @@ -6,4 +6,18 @@ Django 3.2.19 release notes Django 3.2.19 fixes a security issue with severity "low" in 3.2.18. -... +CVE-2023-31047: Potential bypass of validation when uploading multiple files using one form field +================================================================================================= + +Uploading multiple files using one form field has never been supported by +:class:`.forms.FileField` or :class:`.forms.ImageField` as only the last +uploaded file was validated. Unfortunately, :ref:`uploading_multiple_files` +topic suggested otherwise. + +In order to avoid the vulnerability, :class:`~django.forms.ClearableFileInput` +and :class:`~django.forms.FileInput` form widgets now raise ``ValueError`` when +the ``multiple`` HTML attribute is set on them. To prevent the exception and +keep the old behavior, set ``allow_multiple_selected`` to ``True``. + +For more details on using the new attribute and handling of multiple files +through a single field, see :ref:`uploading_multiple_files`. diff --git a/docs/topics/http/file-uploads.txt b/docs/topics/http/file-uploads.txt index 7ba6db00bf7..4205b0a1e40 100644 --- a/docs/topics/http/file-uploads.txt +++ b/docs/topics/http/file-uploads.txt @@ -126,19 +126,54 @@ model:: form = UploadFileForm() return render(request, 'upload.html', {'form': form}) +.. _uploading_multiple_files: + Uploading multiple files ------------------------ -If you want to upload multiple files using one form field, set the ``multiple`` -HTML attribute of field's widget: +.. + Tests in tests.forms_tests.field_tests.test_filefield.MultipleFileFieldTest + should be updated after any changes in the following snippets. + +If you want to upload multiple files using one form field, create a subclass +of the field's widget and set the ``allow_multiple_selected`` attribute on it +to ``True``. + +In order for such files to be all validated by your form (and have the value of +the field include them all), you will also have to subclass ``FileField``. See +below for an example. + +.. admonition:: Multiple file field + + Django is likely to have a proper multiple file field support at some point + in the future. .. code-block:: python :caption: ``forms.py`` from django import forms + + class MultipleFileInput(forms.ClearableFileInput): + allow_multiple_selected = True + + + class MultipleFileField(forms.FileField): + def __init__(self, *args, **kwargs): + kwargs.setdefault("widget", MultipleFileInput()) + super().__init__(*args, **kwargs) + + def clean(self, data, initial=None): + single_file_clean = super().clean + if isinstance(data, (list, tuple)): + result = [single_file_clean(d, initial) for d in data] + else: + result = single_file_clean(data, initial) + return result + + class FileFieldForm(forms.Form): - file_field = forms.FileField(widget=forms.ClearableFileInput(attrs={'multiple': True})) + file_field = MultipleFileField() Then override the ``post`` method of your :class:`~django.views.generic.edit.FormView` subclass to handle multiple file @@ -158,14 +193,32 @@ uploads: def post(self, request, *args, **kwargs): form_class = self.get_form_class() form = self.get_form(form_class) - files = request.FILES.getlist('file_field') if form.is_valid(): - for f in files: - ... # Do something with each file. return self.form_valid(form) else: return self.form_invalid(form) + def form_valid(self, form): + files = form.cleaned_data["file_field"] + for f in files: + ... # Do something with each file. + return super().form_valid() + +.. warning:: + + This will allow you to handle multiple files at the form level only. Be + aware that you cannot use it to put multiple files on a single model + instance (in a single field), for example, even if the custom widget is used + with a form field related to a model ``FileField``. + +.. versionchanged:: 3.2.19 + + In previous versions, there was no support for the ``allow_multiple_selected`` + class attribute, and users were advised to create the widget with the HTML + attribute ``multiple`` set through the ``attrs`` argument. However, this + caused validation of the form field to be applied only to the last file + submitted, which could have adverse security implications. + Upload Handlers =============== diff --git a/tests/forms_tests/field_tests/test_filefield.py b/tests/forms_tests/field_tests/test_filefield.py index 2db106e4a0d..b54febd5887 100644 --- a/tests/forms_tests/field_tests/test_filefield.py +++ b/tests/forms_tests/field_tests/test_filefield.py @@ -2,7 +2,8 @@ from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile -from django.forms import FileField +from django.core.validators import validate_image_file_extension +from django.forms import FileField, FileInput from django.test import SimpleTestCase @@ -83,3 +84,68 @@ def test_disabled_has_changed(self): def test_file_picklable(self): self.assertIsInstance(pickle.loads(pickle.dumps(FileField())), FileField) + + +class MultipleFileInput(FileInput): + allow_multiple_selected = True + + +class MultipleFileField(FileField): + def __init__(self, *args, **kwargs): + kwargs.setdefault("widget", MultipleFileInput()) + super().__init__(*args, **kwargs) + + def clean(self, data, initial=None): + single_file_clean = super().clean + if isinstance(data, (list, tuple)): + result = [single_file_clean(d, initial) for d in data] + else: + result = single_file_clean(data, initial) + return result + + +class MultipleFileFieldTest(SimpleTestCase): + def test_file_multiple(self): + f = MultipleFileField() + files = [ + SimpleUploadedFile("name1", b"Content 1"), + SimpleUploadedFile("name2", b"Content 2"), + ] + self.assertEqual(f.clean(files), files) + + def test_file_multiple_empty(self): + f = MultipleFileField() + files = [ + SimpleUploadedFile("empty", b""), + SimpleUploadedFile("nonempty", b"Some Content"), + ] + msg = "'The submitted file is empty.'" + with self.assertRaisesMessage(ValidationError, msg): + f.clean(files) + with self.assertRaisesMessage(ValidationError, msg): + f.clean(files[::-1]) + + def test_file_multiple_validation(self): + f = MultipleFileField(validators=[validate_image_file_extension]) + + good_files = [ + SimpleUploadedFile("image1.jpg", b"fake JPEG"), + SimpleUploadedFile("image2.png", b"faux image"), + SimpleUploadedFile("image3.bmp", b"fraudulent bitmap"), + ] + self.assertEqual(f.clean(good_files), good_files) + + evil_files = [ + SimpleUploadedFile("image1.sh", b"#!/bin/bash -c 'echo pwned!'\n"), + SimpleUploadedFile("image2.png", b"faux image"), + SimpleUploadedFile("image3.jpg", b"fake JPEG"), + ] + + evil_rotations = ( + evil_files[i:] + evil_files[:i] # Rotate by i. + for i in range(len(evil_files)) + ) + msg = "File extension “sh” is not allowed. Allowed extensions are: " + for rotated_evil_files in evil_rotations: + with self.assertRaisesMessage(ValidationError, msg): + f.clean(rotated_evil_files) diff --git a/tests/forms_tests/widget_tests/test_clearablefileinput.py b/tests/forms_tests/widget_tests/test_clearablefileinput.py index dee44c42392..6cf1476cb20 100644 --- a/tests/forms_tests/widget_tests/test_clearablefileinput.py +++ b/tests/forms_tests/widget_tests/test_clearablefileinput.py @@ -176,3 +176,8 @@ def test_value_omitted_from_data(self): self.assertIs(widget.value_omitted_from_data({}, {}, 'field'), True) self.assertIs(widget.value_omitted_from_data({}, {'field': 'x'}, 'field'), False) self.assertIs(widget.value_omitted_from_data({'field-clear': 'y'}, {}, 'field'), False) + + def test_multiple_error(self): + msg = "ClearableFileInput doesn't support uploading multiple files." + with self.assertRaisesMessage(ValueError, msg): + ClearableFileInput(attrs={"multiple": True}) diff --git a/tests/forms_tests/widget_tests/test_fileinput.py b/tests/forms_tests/widget_tests/test_fileinput.py index 8eec26253af..8068f70b3b6 100644 --- a/tests/forms_tests/widget_tests/test_fileinput.py +++ b/tests/forms_tests/widget_tests/test_fileinput.py @@ -1,4 +1,6 @@ +from django.core.files.uploadedfile import SimpleUploadedFile from django.forms import FileInput +from django.utils.datastructures import MultiValueDict from .base import WidgetTest @@ -24,3 +26,45 @@ def test_use_required_attribute(self): # user to keep the existing, initial value. self.assertIs(self.widget.use_required_attribute(None), True) self.assertIs(self.widget.use_required_attribute('resume.txt'), False) + + def test_multiple_error(self): + msg = "FileInput doesn't support uploading multiple files." + with self.assertRaisesMessage(ValueError, msg): + FileInput(attrs={"multiple": True}) + + def test_value_from_datadict_multiple(self): + class MultipleFileInput(FileInput): + allow_multiple_selected = True + + file_1 = SimpleUploadedFile("something1.txt", b"content 1") + file_2 = SimpleUploadedFile("something2.txt", b"content 2") + # Uploading multiple files is allowed. + widget = MultipleFileInput(attrs={"multiple": True}) + value = widget.value_from_datadict( + data={"name": "Test name"}, + files=MultiValueDict({"myfile": [file_1, file_2]}), + name="myfile", + ) + self.assertEqual(value, [file_1, file_2]) + # Uploading multiple files is not allowed. + widget = FileInput() + value = widget.value_from_datadict( + data={"name": "Test name"}, + files=MultiValueDict({"myfile": [file_1, file_2]}), + name="myfile", + ) + self.assertEqual(value, file_2) + + def test_multiple_default(self): + class MultipleFileInput(FileInput): + allow_multiple_selected = True + + tests = [ + (None, True), + ({"class": "myclass"}, True), + ({"multiple": False}, False), + ] + for attrs, expected in tests: + with self.subTest(attrs=attrs): + widget = MultipleFileInput(attrs=attrs) + self.assertIs(widget.attrs["multiple"], expected) From fc42edd2e63e89a80e7ca81486291f74359ef8be Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 3 May 2023 13:59:19 +0200 Subject: [PATCH 22/44] [3.2.x] Bumped version for 3.2.19 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 396186c816e..470a9c171c0 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 19, 'alpha', 0) +VERSION = (3, 2, 19, 'final', 0) __version__ = get_version(VERSION) From 15f90ebff35c605293dcbe9541da90f50288aa25 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 3 May 2023 14:00:58 +0200 Subject: [PATCH 23/44] [3.2.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 470a9c171c0..f0080fe833e 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 19, 'final', 0) +VERSION = (3, 2, 20, 'alpha', 0) __version__ = get_version(VERSION) From 47ef12e69cfdeeba3388881d2a7234ab3844115e Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 3 May 2023 15:20:31 +0200 Subject: [PATCH 24/44] [3.2.x] Added CVE-2023-31047 to security archive. Backport of 49830025c992fbc8d8f213e7c16dba1391c6adf2 from main --- docs/releases/security.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/releases/security.txt b/docs/releases/security.txt index 0a827387094..c90d7323978 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -36,6 +36,17 @@ Issues under Django's security process All security issues have been handled under versions of Django's security process. These are listed below. +May 3, 2023 - :cve:`2023-31047` +------------------------------- + +Potential bypass of validation when uploading multiple files using one form +field. `Full description +`__ + +* Django 4.2 :commit:`(patch) <21b1b1fc03e5f9e9f8c977ee6e35618dd3b353dd>` +* Django 4.1 :commit:`(patch) ` +* Django 3.2 :commit:`(patch) ` + February 14, 2023 - :cve:`2023-24580` ------------------------------------- From e1bbbbe6acb69e755554088bc573cc1835673209 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Thu, 4 May 2023 08:09:02 +0200 Subject: [PATCH 25/44] [3.2.x] Fixed MultipleFileFieldTest.test_file_multiple_validation() test if Pillow isn't installed. Follow up to fb4c55d9ec4bb812a7fb91fa20510d91645e411b. Backport of fcfbf08abe3e6dc54894df6988024f055abc6c40 from main --- tests/forms_tests/field_tests/test_filefield.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/forms_tests/field_tests/test_filefield.py b/tests/forms_tests/field_tests/test_filefield.py index b54febd5887..b04a70a7596 100644 --- a/tests/forms_tests/field_tests/test_filefield.py +++ b/tests/forms_tests/field_tests/test_filefield.py @@ -1,4 +1,5 @@ import pickle +import unittest from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile @@ -6,6 +7,13 @@ from django.forms import FileField, FileInput from django.test import SimpleTestCase +try: + from PIL import Image # NOQA +except ImportError: + HAS_PILLOW = False +else: + HAS_PILLOW = True + class FileFieldTest(SimpleTestCase): @@ -125,6 +133,7 @@ def test_file_multiple_empty(self): with self.assertRaisesMessage(ValidationError, msg): f.clean(files[::-1]) + @unittest.skipUnless(HAS_PILLOW, "Pillow not installed") def test_file_multiple_validation(self): f = MultipleFileField(validators=[validate_image_file_extension]) From 07cc014cb30bc3c343a25c81aad6820bbc72c0d9 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 21 Jun 2023 08:59:10 +0200 Subject: [PATCH 26/44] [3.2.x] Added stub release notes for 3.2.20. Backport of 2360ba22742c3ee8729697bfe2d508110465af56 from main --- docs/releases/3.2.20.txt | 9 +++++++++ docs/releases/index.txt | 1 + 2 files changed, 10 insertions(+) create mode 100644 docs/releases/3.2.20.txt diff --git a/docs/releases/3.2.20.txt b/docs/releases/3.2.20.txt new file mode 100644 index 00000000000..e4ef9143940 --- /dev/null +++ b/docs/releases/3.2.20.txt @@ -0,0 +1,9 @@ +=========================== +Django 3.2.20 release notes +=========================== + +*July 3, 2023* + +Django 3.2.20 fixes a security issue with severity "moderate" in 3.2.19. + +... diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 0b045b730aa..1da0a752b32 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.2.20 3.2.19 3.2.18 3.2.17 From 454f2fb93437f98917283336201b4048293f7582 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 14 Jun 2023 12:23:06 +0200 Subject: [PATCH 27/44] [3.2.x] Fixed CVE-2023-36053 -- Prevented potential ReDoS in EmailValidator and URLValidator. Thanks Seokchan Yoon for reports. --- django/core/validators.py | 7 ++++-- django/forms/fields.py | 3 +++ docs/ref/forms/fields.txt | 7 +++++- docs/ref/validators.txt | 25 ++++++++++++++++++- docs/releases/3.2.20.txt | 7 +++++- .../field_tests/test_emailfield.py | 5 +++- tests/forms_tests/tests/test_forms.py | 19 +++++++++----- tests/validators/tests.py | 11 ++++++++ 8 files changed, 72 insertions(+), 12 deletions(-) diff --git a/django/core/validators.py b/django/core/validators.py index 731ccf2d469..b9b58dfa617 100644 --- a/django/core/validators.py +++ b/django/core/validators.py @@ -93,6 +93,7 @@ class URLValidator(RegexValidator): message = _('Enter a valid URL.') schemes = ['http', 'https', 'ftp', 'ftps'] unsafe_chars = frozenset('\t\r\n') + max_length = 2048 def __init__(self, schemes=None, **kwargs): super().__init__(**kwargs) @@ -100,7 +101,7 @@ def __init__(self, schemes=None, **kwargs): self.schemes = schemes def __call__(self, value): - if not isinstance(value, str): + if not isinstance(value, str) or len(value) > self.max_length: raise ValidationError(self.message, code=self.code, params={'value': value}) if self.unsafe_chars.intersection(value): raise ValidationError(self.message, code=self.code, params={'value': value}) @@ -210,7 +211,9 @@ def __init__(self, message=None, code=None, allowlist=None, *, whitelist=None): self.domain_allowlist = allowlist def __call__(self, value): - if not value or '@' not in value: + # The maximum length of an email is 320 characters per RFC 3696 + # section 3. + if not value or '@' not in value or len(value) > 320: raise ValidationError(self.message, code=self.code, params={'value': value}) user_part, domain_part = value.rsplit('@', 1) diff --git a/django/forms/fields.py b/django/forms/fields.py index 0214d60c1cf..8adb09e3829 100644 --- a/django/forms/fields.py +++ b/django/forms/fields.py @@ -540,6 +540,9 @@ class EmailField(CharField): default_validators = [validators.validate_email] def __init__(self, **kwargs): + # The default maximum length of an email is 320 characters per RFC 3696 + # section 3. + kwargs.setdefault("max_length", 320) super().__init__(strip=True, **kwargs) diff --git a/docs/ref/forms/fields.txt b/docs/ref/forms/fields.txt index 9438214a28c..5b485f21538 100644 --- a/docs/ref/forms/fields.txt +++ b/docs/ref/forms/fields.txt @@ -592,7 +592,12 @@ For each field, we describe the default widget used if you don't specify * Error message keys: ``required``, ``invalid`` Has three optional arguments ``max_length``, ``min_length``, and - ``empty_value`` which work just as they do for :class:`CharField`. + ``empty_value`` which work just as they do for :class:`CharField`. The + ``max_length`` argument defaults to 320 (see :rfc:`3696#section-3`). + + .. versionchanged:: 3.2.20 + + The default value for ``max_length`` was changed to 320 characters. ``FileField`` ------------- diff --git a/docs/ref/validators.txt b/docs/ref/validators.txt index 50761e5a425..b22762b17b9 100644 --- a/docs/ref/validators.txt +++ b/docs/ref/validators.txt @@ -130,6 +130,11 @@ to, or in lieu of custom ``field.clean()`` methods. :param code: If not ``None``, overrides :attr:`code`. :param allowlist: If not ``None``, overrides :attr:`allowlist`. + An :class:`EmailValidator` ensures that a value looks like an email, and + raises a :exc:`~django.core.exceptions.ValidationError` with + :attr:`message` and :attr:`code` if it doesn't. Values longer than 320 + characters are always considered invalid. + .. attribute:: message The error message used by @@ -158,13 +163,19 @@ to, or in lieu of custom ``field.clean()`` methods. The undocumented ``domain_whitelist`` attribute is deprecated. Use ``domain_allowlist`` instead. + .. versionchanged:: 3.2.20 + + In older versions, values longer than 320 characters could be + considered valid. + ``URLValidator`` ---------------- .. class:: URLValidator(schemes=None, regex=None, message=None, code=None) A :class:`RegexValidator` subclass that ensures a value looks like a URL, - and raises an error code of ``'invalid'`` if it doesn't. + and raises an error code of ``'invalid'`` if it doesn't. Values longer than + :attr:`max_length` characters are always considered invalid. Loopback addresses and reserved IP spaces are considered valid. Literal IPv6 addresses (:rfc:`3986#section-3.2.2`) and Unicode domains are both @@ -181,6 +192,18 @@ to, or in lieu of custom ``field.clean()`` methods. .. _valid URI schemes: https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml + .. attribute:: max_length + + .. versionadded:: 3.2.20 + + The maximum length of values that could be considered valid. Defaults + to 2048 characters. + + .. versionchanged:: 3.2.20 + + In older versions, values longer than 2048 characters could be + considered valid. + ``validate_email`` ------------------ diff --git a/docs/releases/3.2.20.txt b/docs/releases/3.2.20.txt index e4ef9143940..c8f60a70e2d 100644 --- a/docs/releases/3.2.20.txt +++ b/docs/releases/3.2.20.txt @@ -6,4 +6,9 @@ Django 3.2.20 release notes Django 3.2.20 fixes a security issue with severity "moderate" in 3.2.19. -... +CVE-2023-36053: Potential regular expression denial of service vulnerability in ``EmailValidator``/``URLValidator`` +=================================================================================================================== + +``EmailValidator`` and ``URLValidator`` were subject to potential regular +expression denial of service attack via a very large number of domain name +labels of emails and URLs. diff --git a/tests/forms_tests/field_tests/test_emailfield.py b/tests/forms_tests/field_tests/test_emailfield.py index 8b85e4dcc14..19d315205d7 100644 --- a/tests/forms_tests/field_tests/test_emailfield.py +++ b/tests/forms_tests/field_tests/test_emailfield.py @@ -9,7 +9,10 @@ class EmailFieldTest(FormFieldAssertionsMixin, SimpleTestCase): def test_emailfield_1(self): f = EmailField() - self.assertWidgetRendersTo(f, '') + self.assertEqual(f.max_length, 320) + self.assertWidgetRendersTo( + f, '' + ) with self.assertRaisesMessage(ValidationError, "'This field is required.'"): f.clean('') with self.assertRaisesMessage(ValidationError, "'This field is required.'"): diff --git a/tests/forms_tests/tests/test_forms.py b/tests/forms_tests/tests/test_forms.py index 26f8ecafea4..82a32af403a 100644 --- a/tests/forms_tests/tests/test_forms.py +++ b/tests/forms_tests/tests/test_forms.py @@ -422,11 +422,18 @@ class SignupForm(Form): get_spam = BooleanField() f = SignupForm(auto_id=False) - self.assertHTMLEqual(str(f['email']), '') + self.assertHTMLEqual( + str(f["email"]), + '', + ) self.assertHTMLEqual(str(f['get_spam']), '') f = SignupForm({'email': 'test@example.com', 'get_spam': True}, auto_id=False) - self.assertHTMLEqual(str(f['email']), '') + self.assertHTMLEqual( + str(f["email"]), + '", + ) self.assertHTMLEqual( str(f['get_spam']), '', @@ -2824,7 +2831,7 @@ class Person(Form): -
    • +
    • This field is required.
    • """ ) @@ -2840,7 +2847,7 @@ class Person(Form):

    -

    +

    • This field is required.

    """ @@ -2859,7 +2866,7 @@ class Person(Form): - +
    • This field is required.
    """ @@ -3489,7 +3496,7 @@ class CommentForm(Form): f = CommentForm(data, auto_id=False, error_class=DivErrorList) self.assertHTMLEqual(f.as_p(), """

    Name:

    Enter a valid email address.
    -

    Email:

    +

    Email:

    This field is required.

    Comment:

    """) diff --git a/tests/validators/tests.py b/tests/validators/tests.py index e39d0e3a1ce..1065727a974 100644 --- a/tests/validators/tests.py +++ b/tests/validators/tests.py @@ -59,6 +59,7 @@ (validate_email, 'example@atm.%s' % ('a' * 64), ValidationError), (validate_email, 'example@%s.atm.%s' % ('b' * 64, 'a' * 63), ValidationError), + (validate_email, "example@%scom" % (("a" * 63 + ".") * 100), ValidationError), (validate_email, None, ValidationError), (validate_email, '', ValidationError), (validate_email, 'abc', ValidationError), @@ -246,6 +247,16 @@ (URLValidator(), None, ValidationError), (URLValidator(), 56, ValidationError), (URLValidator(), 'no_scheme', ValidationError), + ( + URLValidator(), + "http://example." + ("a" * 63 + ".") * 1000 + "com", + ValidationError, + ), + ( + URLValidator(), + "http://userid:password" + "d" * 2000 + "@example.aaaaaaaaaaaaa.com", + None, + ), # Newlines and tabs are not accepted. (URLValidator(), 'http://www.djangoproject.com/\n', ValidationError), (URLValidator(), 'http://[::ffff:192.9.5.5]\n', ValidationError), From 19bc11f636ca2b5b80c3d9ad5b489e43abad52bb Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 3 Jul 2023 08:33:38 +0200 Subject: [PATCH 28/44] [3.2.x] Bumped version for 3.2.20 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index f0080fe833e..241cd99c8b3 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 20, 'alpha', 0) +VERSION = (3, 2, 20, 'final', 0) __version__ = get_version(VERSION) From 4012a87a58292f34e72181d9f12c94317ac6e911 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 3 Jul 2023 08:36:12 +0200 Subject: [PATCH 29/44] [3.2.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 241cd99c8b3..c2c0da8b793 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 20, 'final', 0) +VERSION = (3, 2, 21, 'alpha', 0) __version__ = get_version(VERSION) From 848fe70f3ee6dc151831251076dc0a4a9db5a0ec Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 3 Jul 2023 10:10:48 +0200 Subject: [PATCH 30/44] [3.2.x] Added CVE-2023-36053 to security archive. Backport of 1d6fbf16f24200a556beb6dd197439944deb6837 from main --- docs/releases/security.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/releases/security.txt b/docs/releases/security.txt index c90d7323978..48586c8a6e9 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -36,6 +36,17 @@ Issues under Django's security process All security issues have been handled under versions of Django's security process. These are listed below. +July 3, 2023 - :cve:`2023-36053` +-------------------------------- + +Potential regular expression denial of service vulnerability in +``EmailValidator``/``URLValidator``. `Full description +`__ + +* Django 4.2 :commit:`(patch) ` +* Django 4.1 :commit:`(patch) ` +* Django 3.2 :commit:`(patch) <454f2fb93437f98917283336201b4048293f7582>` + May 3, 2023 - :cve:`2023-31047` ------------------------------- From 75418f8c0e5e8fcfb557bf8e69fc1ccfd64b697d Mon Sep 17 00:00:00 2001 From: David Smith Date: Tue, 1 Aug 2023 21:22:53 +0100 Subject: [PATCH 31/44] [3.2.x] Fixed #34756 -- Fixed docs HTML build on Sphinx 7.1+. Backport of b3e0170ab546a96930ce3114b0a1a560953c0ff4 from main --- docs/_ext/djangodocs.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/_ext/djangodocs.py b/docs/_ext/djangodocs.py index 5208a532554..6eadc23144d 100644 --- a/docs/_ext/djangodocs.py +++ b/docs/_ext/djangodocs.py @@ -130,10 +130,23 @@ def depart_table(self, node): def visit_desc_parameterlist(self, node): self.body.append('(') # by default sphinx puts around the "(" - self.first_param = 1 self.optional_param_level = 0 self.param_separator = node.child_text_separator - self.required_params_left = sum(isinstance(c, addnodes.desc_parameter) for c in node.children) + # Counts 'parameter groups' being either a required parameter, or a set + # of contiguous optional ones. + required_params = [ + isinstance(c, addnodes.desc_parameter) for c in node.children + ] + # How many required parameters are left. + self.required_params_left = sum(required_params) + if sphinx_version < (7, 1): + self.first_param = 1 + else: + self.is_first_param = True + self.params_left_at_level = 0 + self.param_group_index = 0 + self.list_is_required_param = required_params + self.multi_line_parameter_list = False def depart_desc_parameterlist(self, node): self.body.append(')') From 73350a63698199c9f1269647722ea96c7f9a8aa0 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Thu, 24 Aug 2023 07:56:05 +0200 Subject: [PATCH 32/44] [3.2.x] Added stub release notes for 3.2.21. Backport of 24f1a38b37c0af3a5ce0dd7b5392fe4e75d7e1dc from main. --- docs/releases/3.2.21.txt | 9 +++++++++ docs/releases/index.txt | 1 + 2 files changed, 10 insertions(+) create mode 100644 docs/releases/3.2.21.txt diff --git a/docs/releases/3.2.21.txt b/docs/releases/3.2.21.txt new file mode 100644 index 00000000000..79efc679d17 --- /dev/null +++ b/docs/releases/3.2.21.txt @@ -0,0 +1,9 @@ +=========================== +Django 3.2.21 release notes +=========================== + +*September 4, 2023* + +Django 3.2.21 fixes a security issue with severity "moderate" in 3.2.20. + +... diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 1da0a752b32..e0ae1b15803 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.2.21 3.2.20 3.2.19 3.2.18 From 6f030b1149bd8fa4ba90452e77cb3edc095ce54e Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 22 Aug 2023 08:53:03 +0200 Subject: [PATCH 33/44] [3.2.x] Fixed CVE-2023-41164 -- Fixed potential DoS in django.utils.encoding.uri_to_iri(). Thanks MProgrammer (https://hackerone.com/mprogrammer) for the report. Co-authored-by: nessita <124304+nessita@users.noreply.github.com> --- django/utils/encoding.py | 6 ++++-- docs/releases/3.2.21.txt | 7 ++++++- tests/utils_tests/test_encoding.py | 21 ++++++++++++++++++++- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/django/utils/encoding.py b/django/utils/encoding.py index e1ebacef470..c5c4463b1c2 100644 --- a/django/utils/encoding.py +++ b/django/utils/encoding.py @@ -229,6 +229,7 @@ def repercent_broken_unicode(path): repercent-encode any octet produced that is not part of a strictly legal UTF-8 octet sequence. """ + changed_parts = [] while True: try: path.decode() @@ -236,9 +237,10 @@ def repercent_broken_unicode(path): # CVE-2019-14235: A recursion shouldn't be used since the exception # handling uses massive amounts of memory repercent = quote(path[e.start:e.end], safe=b"/#%[]=:;$&()+,!?*@'~") - path = path[:e.start] + repercent.encode() + path[e.end:] + changed_parts.append(path[:e.start] + repercent.encode()) + path = path[e.end:] else: - return path + return b"".join(changed_parts) + path def filepath_to_uri(path): diff --git a/docs/releases/3.2.21.txt b/docs/releases/3.2.21.txt index 79efc679d17..062ac66682a 100644 --- a/docs/releases/3.2.21.txt +++ b/docs/releases/3.2.21.txt @@ -6,4 +6,9 @@ Django 3.2.21 release notes Django 3.2.21 fixes a security issue with severity "moderate" in 3.2.20. -... +CVE-2023-41164: Potential denial of service vulnerability in ``django.utils.encoding.uri_to_iri()`` +=================================================================================================== + +``django.utils.encoding.uri_to_iri()`` was subject to potential denial of +service attack via certain inputs with a very large number of Unicode +characters. diff --git a/tests/utils_tests/test_encoding.py b/tests/utils_tests/test_encoding.py index 36f2d8665f3..42779050cb3 100644 --- a/tests/utils_tests/test_encoding.py +++ b/tests/utils_tests/test_encoding.py @@ -1,9 +1,10 @@ import datetime +import inspect import sys import unittest from pathlib import Path from unittest import mock -from urllib.parse import quote_plus +from urllib.parse import quote, quote_plus from django.test import SimpleTestCase from django.utils.encoding import ( @@ -101,6 +102,24 @@ def test_repercent_broken_unicode_recursion_error(self): except RecursionError: self.fail('Unexpected RecursionError raised.') + def test_repercent_broken_unicode_small_fragments(self): + data = b"test\xfctest\xfctest\xfc" + decoded_paths = [] + + def mock_quote(*args, **kwargs): + # The second frame is the call to repercent_broken_unicode(). + decoded_paths.append(inspect.currentframe().f_back.f_locals["path"]) + return quote(*args, **kwargs) + + with mock.patch("django.utils.encoding.quote", mock_quote): + self.assertEqual(repercent_broken_unicode(data), b"test%FCtest%FCtest%FC") + + # decode() is called on smaller fragment of the path each time. + self.assertEqual( + decoded_paths, + [b"test\xfctest\xfctest\xfc", b"test\xfctest\xfc", b"test\xfc"], + ) + class TestRFC3987IEncodingUtils(unittest.TestCase): From fd0ccd7fb3ec8aee9a88ab0576059c7393397c8d Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 4 Sep 2023 12:23:57 +0200 Subject: [PATCH 34/44] [3.2.x] Bumped version for 3.2.21 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index c2c0da8b793..48e561f860f 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 21, 'alpha', 0) +VERSION = (3, 2, 21, 'final', 0) __version__ = get_version(VERSION) From 4b439dcd05ba1fbd0202382affa4df71111831da Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 4 Sep 2023 12:25:28 +0200 Subject: [PATCH 35/44] [3.2.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 48e561f860f..919d1ac989a 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 21, 'final', 0) +VERSION = (3, 2, 22, 'alpha', 0) __version__ = get_version(VERSION) From 9e814c3a5e399cb5fa3ea2f0f3b9536dc7302fea Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 4 Sep 2023 13:09:48 +0200 Subject: [PATCH 36/44] [3.2.x] Added CVE-2023-41164 to security archive. Backport of 8a98768868a104ea3ce10d8182590bdd095d9ccb from main --- docs/releases/security.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/releases/security.txt b/docs/releases/security.txt index 48586c8a6e9..34394c50b0b 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -36,6 +36,17 @@ Issues under Django's security process All security issues have been handled under versions of Django's security process. These are listed below. +September 4, 2023 - :cve:`2023-41164` +------------------------------------- + +Potential denial of service vulnerability in +``django.utils.encoding.uri_to_iri()``. `Full description +`__ + +* Django 4.2 :commit:`(patch) <9c51b4dcfa0cefcb48231f4d71cafa80821f87b9>` +* Django 4.1 :commit:`(patch) ` +* Django 3.2 :commit:`(patch) <6f030b1149bd8fa4ba90452e77cb3edc095ce54e>` + July 3, 2023 - :cve:`2023-36053` -------------------------------- From 6caf7b313d279d0002bc27b81a92c0bf7cc86e41 Mon Sep 17 00:00:00 2001 From: Natalia <124304+nessita@users.noreply.github.com> Date: Wed, 27 Sep 2023 14:18:40 -0300 Subject: [PATCH 37/44] [3.2.x] Added stub release notes for 3.2.22. Backport of 24f1a38b37c0af3a5ce0dd7b5392fe4e75d7e1dc from main. --- docs/releases/3.2.22.txt | 9 +++++++++ docs/releases/index.txt | 1 + 2 files changed, 10 insertions(+) create mode 100644 docs/releases/3.2.22.txt diff --git a/docs/releases/3.2.22.txt b/docs/releases/3.2.22.txt new file mode 100644 index 00000000000..6e1815de11d --- /dev/null +++ b/docs/releases/3.2.22.txt @@ -0,0 +1,9 @@ +=========================== +Django 3.2.22 release notes +=========================== + +*October 4, 2023* + +Django 3.2.22 fixes a security issue with severity "moderate" in 3.2.21. + +... diff --git a/docs/releases/index.txt b/docs/releases/index.txt index e0ae1b15803..b35f225b53f 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.2.22 3.2.21 3.2.20 3.2.19 From ccdade1a0262537868d7ca64374de3d957ca50c5 Mon Sep 17 00:00:00 2001 From: Natalia <124304+nessita@users.noreply.github.com> Date: Tue, 19 Sep 2023 09:51:48 -0300 Subject: [PATCH 38/44] [3.2.x] Fixed CVE-2023-43665 -- Mitigated potential DoS in django.utils.text.Truncator when truncating HTML text. Thanks Wenchao Li of Alibaba Group for the report. --- django/utils/text.py | 18 ++++++++++++++++- docs/ref/templates/builtins.txt | 20 +++++++++++++++++++ docs/releases/3.2.22.txt | 18 ++++++++++++++++- tests/utils_tests/test_text.py | 35 ++++++++++++++++++++++++--------- 4 files changed, 80 insertions(+), 11 deletions(-) diff --git a/django/utils/text.py b/django/utils/text.py index baa44f279e8..83e258fa81c 100644 --- a/django/utils/text.py +++ b/django/utils/text.py @@ -60,7 +60,14 @@ def _generator(): class Truncator(SimpleLazyObject): """ An object used to truncate text, either by characters or words. + + When truncating HTML text (either chars or words), input will be limited to + at most `MAX_LENGTH_HTML` characters. """ + + # 5 million characters are approximately 4000 text pages or 3 web pages. + MAX_LENGTH_HTML = 5_000_000 + def __init__(self, text): super().__init__(lambda: str(text)) @@ -157,6 +164,11 @@ def _truncate_html(self, length, truncate, text, truncate_len, words): if words and length <= 0: return '' + size_limited = False + if len(text) > self.MAX_LENGTH_HTML: + text = text[: self.MAX_LENGTH_HTML] + size_limited = True + html4_singlets = ( 'br', 'col', 'link', 'base', 'img', 'param', 'area', 'hr', 'input' @@ -206,10 +218,14 @@ def _truncate_html(self, length, truncate, text, truncate_len, words): # Add it to the start of the open tags list open_tags.insert(0, tagname) + truncate_text = self.add_truncation_text("", truncate) + if current_len <= length: + if size_limited and truncate_text: + text += truncate_text return text + out = text[:end_text_pos] - truncate_text = self.add_truncation_text('', truncate) if truncate_text: out += truncate_text # Close any tags still open diff --git a/docs/ref/templates/builtins.txt b/docs/ref/templates/builtins.txt index 22509a2a7e0..a6fd97175b7 100644 --- a/docs/ref/templates/builtins.txt +++ b/docs/ref/templates/builtins.txt @@ -2348,6 +2348,16 @@ If ``value`` is ``"

    Joel is a slug

    "``, the output will be Newlines in the HTML content will be preserved. +.. admonition:: Size of input string + + Processing large, potentially malformed HTML strings can be + resource-intensive and impact service performance. ``truncatechars_html`` + limits input to the first five million characters. + +.. versionchanged:: 3.2.22 + + In older versions, strings over five million characters were processed. + .. templatefilter:: truncatewords ``truncatewords`` @@ -2386,6 +2396,16 @@ If ``value`` is ``"

    Joel is a slug

    "``, the output will be Newlines in the HTML content will be preserved. +.. admonition:: Size of input string + + Processing large, potentially malformed HTML strings can be + resource-intensive and impact service performance. ``truncatewords_html`` + limits input to the first five million characters. + +.. versionchanged:: 3.2.22 + + In older versions, strings over five million characters were processed. + .. templatefilter:: unordered_list ``unordered_list`` diff --git a/docs/releases/3.2.22.txt b/docs/releases/3.2.22.txt index 6e1815de11d..cfedc41de8b 100644 --- a/docs/releases/3.2.22.txt +++ b/docs/releases/3.2.22.txt @@ -6,4 +6,20 @@ Django 3.2.22 release notes Django 3.2.22 fixes a security issue with severity "moderate" in 3.2.21. -... +CVE-2023-43665: Denial-of-service possibility in ``django.utils.text.Truncator`` +================================================================================ + +Following the fix for :cve:`2019-14232`, the regular expressions used in the +implementation of ``django.utils.text.Truncator``'s ``chars()`` and ``words()`` +methods (with ``html=True``) were revised and improved. However, these regular +expressions still exhibited linear backtracking complexity, so when given a +very long, potentially malformed HTML input, the evaluation would still be +slow, leading to a potential denial of service vulnerability. + +The ``chars()`` and ``words()`` methods are used to implement the +:tfilter:`truncatechars_html` and :tfilter:`truncatewords_html` template +filters, which were thus also vulnerable. + +The input processed by ``Truncator``, when operating in HTML mode, has been +limited to the first five million characters in order to avoid potential +performance and memory issues. diff --git a/tests/utils_tests/test_text.py b/tests/utils_tests/test_text.py index d2a94fcdabe..0a6f0bc3f26 100644 --- a/tests/utils_tests/test_text.py +++ b/tests/utils_tests/test_text.py @@ -1,5 +1,6 @@ import json import sys +from unittest.mock import patch from django.core.exceptions import SuspiciousFileOperation from django.test import SimpleTestCase, ignore_warnings @@ -90,11 +91,17 @@ def test_truncate_chars(self): # lazy strings are handled correctly self.assertEqual(text.Truncator(lazystr('The quick brown fox')).chars(10), 'The quick…') - def test_truncate_chars_html(self): + @patch("django.utils.text.Truncator.MAX_LENGTH_HTML", 10_000) + def test_truncate_chars_html_size_limit(self): + max_len = text.Truncator.MAX_LENGTH_HTML + bigger_len = text.Truncator.MAX_LENGTH_HTML + 1 + valid_html = "

    Joel is a slug

    " # 14 chars perf_test_values = [ - (('', None), - ('&' * 50000, '&' * 9 + '…'), - ('_X<<<<<<<<<<<>', None), + ("", None), + ("", "", None), + (valid_html * bigger_len, "

    Joel is a…

    "), # 10 chars ] for value, expected in perf_test_values: with self.subTest(value=value): @@ -152,15 +159,25 @@ def test_truncate_html_words(self): truncator = text.Truncator('

    I <3 python, what about you?

    ') self.assertEqual('

    I <3 python,…

    ', truncator.words(3, html=True)) + @patch("django.utils.text.Truncator.MAX_LENGTH_HTML", 10_000) + def test_truncate_words_html_size_limit(self): + max_len = text.Truncator.MAX_LENGTH_HTML + bigger_len = text.Truncator.MAX_LENGTH_HTML + 1 + valid_html = "

    Joel is a slug

    " # 4 words perf_test_values = [ - ('', - '&' * 50000, - '_X<<<<<<<<<<<>', + ("", None), + ("", "", None), + (valid_html * bigger_len, valid_html * 12 + "

    Joel is…

    "), # 50 words ] - for value in perf_test_values: + for value, expected in perf_test_values: with self.subTest(value=value): truncator = text.Truncator(value) - self.assertEqual(value, truncator.words(50, html=True)) + self.assertEqual( + expected if expected else value, truncator.words(50, html=True) + ) def test_wrap(self): digits = '1234 67 9' From 3106e94e52a0a7786d49803c86b3139df113d36d Mon Sep 17 00:00:00 2001 From: Natalia <124304+nessita@users.noreply.github.com> Date: Wed, 4 Oct 2023 10:34:56 -0300 Subject: [PATCH 39/44] [3.2.x] Bumped version for 3.2.22 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 919d1ac989a..3fe7f63eab5 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 22, 'alpha', 0) +VERSION = (3, 2, 22, 'final', 0) __version__ = get_version(VERSION) From 86a14d653f205dc94c7fb4f0c7f7a8e5a551a373 Mon Sep 17 00:00:00 2001 From: Natalia <124304+nessita@users.noreply.github.com> Date: Wed, 4 Oct 2023 10:57:29 -0300 Subject: [PATCH 40/44] [3.2.x] Post release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 3fe7f63eab5..9642fa2b39c 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 22, 'final', 0) +VERSION = (3, 2, 23, 'alpha', 0) __version__ = get_version(VERSION) From 3c04b7429369521c513bc57108c82c904ea280b4 Mon Sep 17 00:00:00 2001 From: Natalia <124304+nessita@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:05:00 -0300 Subject: [PATCH 41/44] [3.2.x] Added CVE-2023-43665 to security archive. Backport of 4e790271e3e65c9ad037b347a34fa95e11982228 from main --- docs/releases/security.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/releases/security.txt b/docs/releases/security.txt index 34394c50b0b..6fdee8bf577 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -36,6 +36,17 @@ Issues under Django's security process All security issues have been handled under versions of Django's security process. These are listed below. +October 4, 2023 - :cve:`2023-43665` +------------------------------------- + +Denial-of-service possibility in ``django.utils.text.Truncator``. +`Full description +`__ + +* Django 4.2 :commit:`(patch) ` +* Django 4.1 :commit:`(patch) ` +* Django 3.2 :commit:`(patch) ` + September 4, 2023 - :cve:`2023-41164` ------------------------------------- From e6d2591d9e8a517b891e49de476640ecae93cc41 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 25 Oct 2023 05:37:57 +0200 Subject: [PATCH 42/44] [3.2.x] Added stub release notes for 3.2.23. Backport of fdd1323b9c83e56184e0c992af8faf8d54327775 from main. --- docs/releases/3.2.23.txt | 9 +++++++++ docs/releases/index.txt | 1 + 2 files changed, 10 insertions(+) create mode 100644 docs/releases/3.2.23.txt diff --git a/docs/releases/3.2.23.txt b/docs/releases/3.2.23.txt new file mode 100644 index 00000000000..0d71a0aa716 --- /dev/null +++ b/docs/releases/3.2.23.txt @@ -0,0 +1,9 @@ +=========================== +Django 3.2.23 release notes +=========================== + +*November 1, 2023* + +Django 3.2.23 fixes a security issue with severity "moderate" in 3.2.22. + +... diff --git a/docs/releases/index.txt b/docs/releases/index.txt index b35f225b53f..9d0678ccdc9 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.2.23 3.2.22 3.2.21 3.2.20 From f9a7fb8466a7ba4857eaf930099b5258f3eafb2b Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 17 Oct 2023 11:48:32 +0200 Subject: [PATCH 43/44] [3.2.x] Fixed CVE-2023-46695 -- Fixed potential DoS in UsernameField on Windows. Thanks MProgrammer (https://hackerone.com/mprogrammer) for the report. --- django/contrib/auth/forms.py | 10 +++++++++- docs/releases/3.2.23.txt | 12 +++++++++++- tests/auth_tests/test_forms.py | 8 +++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/django/contrib/auth/forms.py b/django/contrib/auth/forms.py index 20d89227990..fb7cfda2098 100644 --- a/django/contrib/auth/forms.py +++ b/django/contrib/auth/forms.py @@ -62,7 +62,15 @@ def __init__(self, *args, **kwargs): class UsernameField(forms.CharField): def to_python(self, value): - return unicodedata.normalize('NFKC', super().to_python(value)) + value = super().to_python(value) + if self.max_length is not None and len(value) > self.max_length: + # Normalization can increase the string length (e.g. + # "ff" -> "ff", "½" -> "1⁄2") but cannot reduce it, so there is no + # point in normalizing invalid data. Moreover, Unicode + # normalization is very slow on Windows and can be a DoS attack + # vector. + return value + return unicodedata.normalize("NFKC", value) def widget_attrs(self, widget): return { diff --git a/docs/releases/3.2.23.txt b/docs/releases/3.2.23.txt index 0d71a0aa716..ba23d11a718 100644 --- a/docs/releases/3.2.23.txt +++ b/docs/releases/3.2.23.txt @@ -6,4 +6,14 @@ Django 3.2.23 release notes Django 3.2.23 fixes a security issue with severity "moderate" in 3.2.22. -... +CVE-2023-46695: Potential denial of service vulnerability in ``UsernameField`` on Windows +========================================================================================= + +The :func:`NFKC normalization ` is slow on +Windows. As a consequence, ``django.contrib.auth.forms.UsernameField`` was +subject to a potential denial of service attack via certain inputs with a very +large number of Unicode characters. + +In order to avoid the vulnerability, invalid values longer than +``UsernameField.max_length`` are no longer normalized, since they cannot pass +validation anyway. diff --git a/tests/auth_tests/test_forms.py b/tests/auth_tests/test_forms.py index 7a731bedc87..c0e1975c1a9 100644 --- a/tests/auth_tests/test_forms.py +++ b/tests/auth_tests/test_forms.py @@ -5,7 +5,7 @@ from django.contrib.auth.forms import ( AdminPasswordChangeForm, AuthenticationForm, PasswordChangeForm, PasswordResetForm, ReadOnlyPasswordHashField, ReadOnlyPasswordHashWidget, - SetPasswordForm, UserChangeForm, UserCreationForm, + SetPasswordForm, UserChangeForm, UserCreationForm, UsernameField, ) from django.contrib.auth.models import User from django.contrib.auth.signals import user_login_failed @@ -132,6 +132,12 @@ def test_normalize_username(self): self.assertNotEqual(user.username, ohm_username) self.assertEqual(user.username, 'testΩ') # U+03A9 GREEK CAPITAL LETTER OMEGA + def test_invalid_username_no_normalize(self): + field = UsernameField(max_length=254) + # Usernames are not normalized if they are too long. + self.assertEqual(field.to_python("½" * 255), "½" * 255) + self.assertEqual(field.to_python("ff" * 254), "ff" * 254) + def test_duplicate_normalized_unicode(self): """ To prevent almost identical usernames, visually identical but differing From 60e648a7ae033a9c2d1ab23a58402f57a1754df1 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 1 Nov 2023 06:31:51 +0100 Subject: [PATCH 44/44] [3.2.x] Bumped version for 3.2.23 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 9642fa2b39c..138d3c8e273 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 2, 23, 'alpha', 0) +VERSION = (3, 2, 23, 'final', 0) __version__ = get_version(VERSION)