8000 bpo-30458: Disallow control chars in http URLs. (GH-12755) (GH-13154) · python/cpython@7e200e0 · GitHub
[go: up one dir, main page]
Skip to content
8000

Commit 7e200e0

Browse files
hroncokgpshead
hroncok
authored and
gpshead
committed
bpo-30458: Disallow control chars in http URLs. (GH-12755) (GH-13154)
Disallow control chars in http URLs in urllib.urlopen. This addresses a potential security problem for applications that do not sanity check their URLs where http request headers could be injected. Disable https related urllib tests on a build without ssl (GH-13032) These tests require an SSL enabled build. Skip these tests when python is built without SSL to fix test failures. Use http.client.InvalidURL instead of ValueError as the new error case's exception. (GH-13044) Backport Co-Authored-By: Miro Hrončok <miro@hroncok.cz>
1 parent 146010e commit 7e200e0

File tree

4 files changed

+75
-1
lines changed

4 files changed

+75
-1
lines changed

‎Lib/http/client.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,16 @@
140140
_is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch
141141
_is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search
142142

143+
# These characters are not allowed within HTTP URL paths.
144+
# See https://tools.ietf.org/html/rfc3986#section-3.3 and the
145+
# https://tools.ietf.org/html/rfc3986#appendix-A pchar definition.
146+
# Prevents CVE-2019-9740. Includes control characters such as \r\n.
147+
# We don't restrict chars above \x7f as putrequest() limits us to ASCII.
148+
_contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')
149+
# Arguably only these _should_ allowed:
150+
# _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")
151+
# We are more lenient for assumed real world compatibility purposes.
152+
143153
# We always set the Content-Length header for these methods because some
144154
# servers will otherwise respond with a 411
145155
_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
@@ -1101,6 +1111,11 @@ def putrequest(self, method, url, skip_host=False,
11011111
self._method = method
11021112
if not url:
11031113
url = '/'
1114+
# Prevent CVE-2019-9740.
1115+
match = _contains_disallowed_url_pchar_re.search(url)
1116+
if match:
1117+
raise InvalidURL(f"URL can't contain control characters. {url!r} "
1118+
f"(found at least {match.group()!r})")
11041119
request = '%s %s %s' % (method, url, self._http_vsn_str)
11051120

11061121
# Non-ASCII characters should have been eliminated earlier

‎Lib/test/test_urllib.py

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,59 @@ def test_willclose(self):
329329
finally:
330330
self.unfakehttp()
331331

332+
@unittest.skipUnless(ssl, "ssl module required")
333+
def test_url_with_control_char_rejected(self):
334+
for char_no in list(range(0, 0x21)) + [0x7f]:
335+
char = chr(char_no)
336+
schemeless_url = f"//localhost:7777/test{char}/"
337+
self.fakehttp(b&q 8000 uot;HTTP/1.1 200 OK\r\n\r\nHello.")
338+
try:
339+
# We explicitly test urllib.request.urlopen() instead of the top
340+
# level 'def urlopen()' function defined in this... (quite ugly)
341+
# test suite. They use different url opening codepaths. Plain
342+
# urlopen uses FancyURLOpener which goes via a codepath that
343+
# calls urllib.parse.quote() on the URL which makes all of the
344+
# above attempts at injection within the url _path_ safe.
345+
escaped_char_repr = repr(char).replace('\\', r'\\')
346+
InvalidURL = http.client.InvalidURL
347+
with self.assertRaisesRegex(
348+
InvalidURL, f"contain control.*{escaped_char_repr}"):
349+
urllib.request.urlopen(f"http:{schemeless_url}")
350+
with self.assertRaisesRegex(
351+
InvalidURL, f"contain control.*{escaped_char_repr}"):
352+
urllib.request.urlopen(f"https:{schemeless_url}")
353+
# This code path quotes the URL so there is no injection.
354+
resp = urlopen(f"http:{schemeless_url}")
355+
self.assertNotIn(char, resp.geturl())
356+
finally:
357+
self.unfakehttp()
358+
359+
@unittest.skipUnless(ssl, "ssl module required")
360+
def test_url_with_newline_header_injection_rejected(self):
361+
self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.")
362+
host = "localhost:7777?a=1 HTTP/1.1\r\nX-injected: header\r\nTEST: 123"
363+
schemeless_url = "//" + host + ":8080/test/?test=a"
364+
try:
365+
# We explicitly test urllib.request.urlopen() instead of the top
366+
# level 'def urlopen()' function defined in this... (quite ugly)
367+
# test suite. They use different url opening codepaths. Plain
368+
# urlopen uses FancyURLOpener which goes via a codepath that
369+
# calls urllib.parse.quote() on the URL which makes all of the
370+
# above attempts at injection within the url _path_ safe.
371+
InvalidURL = http.client.InvalidURL
372+
with self.assertRaisesRegex(
373+
InvalidURL, r"contain control.*\\r.*(found at least . .)"):
374+
urllib.request.urlopen(f"http:{schemeless_url}")
375+
with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"):
376+
urllib.request.urlopen(f"https:{schemeless_url}")
377+
# This code path quotes the URL so there is no injection.
378+
resp = urlopen(f"http:{schemeless_url}")
379+
self.assertNotIn(' ', resp.geturl())
380+
self.assertNotIn('\r', resp.geturl())
381+
self.assertNotIn('\n', resp.geturl())
382+
finally:
383+
self.unfakehttp()
384+
332385
def test_read_0_9(self):
333386
# "0.9" response accepted (but not "simple responses" without
334387
# a status line)

‎Lib/test/test_xmlrpc.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -945,7 +945,12 @@ def test_unicode_host(self):
945945
def test_partial_post(self):
946946
# Check that a partial POST doesn't make the server loop: issue #14001.
947947
conn = http.client.HTTPConnection(ADDR, PORT)
948-
conn.request('POST', '/RPC2 HTTP/1.0\r\nContent-Length: 100\r\n\r\nbye')
948+
conn.send('POST /RPC2 HTTP/1.0\r\n'
949+
'Content-Length: 100\r\n\r\n'
950+
'bye HTTP/1.1\r\n'
951+
f'Host: {ADDR}:{PORT}\r\n'
952+
'Accept-Encoding: identity\r\n'
953+
'Content-Length: 0\r\n\r\n'.encode('ascii'))
949954
conn.close()
950955

951956
def test_context_manager(self):

‎Misc/NEWS.d/next/Security/2019-04-10-08-53-30.bpo-30458.51E-DA.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Address CVE-2019-9740 by disallowing URL paths with embedded whitespace or control characters through into the underlying http client request. Such potentially malicious header injection URLs now cause an http.client.InvalidURL exception to be raised.

0 commit comments

Comments
 (0)
0