10000 Issue #23215: Multibyte codecs with custom error handlers that ignore… · python/cpython@a1543cd · GitHub
[go: up one dir, main page]
Skip to content

Commit a1543cd

Browse files
serhiy-storchakaserhiy-storchaka
committed
Issue #23215: Multibyte codecs with custom error handlers that ignores errors
consumed too much memory and raised SystemError or MemoryError. Original patch by Aleksi Torhamo.
1 parent a3712a9 commit a1543cd

File tree

3 files changed

+22
-8
lines changed

3 files changed

+22
-8
lines changed

‎Lib/test/test_multibytecodec.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@ def test_errorcallback_longindex(self):
4444
self.assertRaises(IndexError, dec,
4545
b'apple\x92ham\x93spam', 'test.cjktest')
4646

47+
def test_errorcallback_custom_ignore(self):
48+
# Issue #23215: MemoryError with custom error handlers and multibyte codecs
49+
data = 100 * "\udc00"
50+
codecs.register_error("test.ignore", codecs.ignore_errors)
51+
for enc in ALL_CJKENCODINGS:
52+
self.assertEqual(data.encode(enc, "test.ignore"), b'')
53+
4754
def test_codingspec(self):
4855
try:
4956
for enc in ALL_CJKENCODINGS:

‎Misc/NEWS

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ Core and Builtins
1313
Library
1414
-------
1515

16+
- Issue #23215: Multibyte codecs with custom error handlers that ignores errors
17+
consumed too much memory and raised SystemError or MemoryError.
18+
Original patch by Aleksi Torhamo.
19+
1620
- Issue #5700: io.FileIO() called flush() after closing the file.
1721
flush() was not called in close() if closefd=False.
1822

‎Modules/cjkcodecs/multibytecodec.c

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -182,8 +182,10 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
182182
orgsize = PyBytes_GET_SIZE(buf->outobj);
183183
incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);
184184

185-
if (orgsize > PY_SSIZE_T_MAX - incsize)
185+
if (orgsize > PY_SSIZE_T_MAX - incsize) {
186+
PyErr_NoMemory();
186187
return -1;
188+
}
187189

188190
if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1)
189191
return -1;
@@ -194,11 +196,11 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
194196

195197
return 0;
196198
}
197-
#define REQUIRE_ENCODEBUFFER(buf, s) { \
198-
if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \
199+
#define REQUIRE_ENCODEBUFFER(buf, s) do { \
200+
if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \
199201
if (expand_encodebuffer(buf, s) == -1) \
200202
goto errorexit; \
201-
}
203+
} while(0)
202204

203205

204206
/**
@@ -332,10 +334,11 @@ multibytecodec_encerror(MultibyteCodec *codec,
332334

333335
assert(PyBytes_Check(retstr));
334336
retstrsize = PyBytes_GET_SIZE(retstr);
335-
REQUIRE_ENCODEBUFFER(buf, retstrsize);
336-
337-
memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize);
338-
buf->outbuf += retstrsize;
337+
if (retstrsize > 0) {
338+
REQUIRE_ENCODEBUFFER(buf, retstrsize);
339+
memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize);
340+
buf->outbuf += retstrsize;
341+
}
339342

340343
newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
341344
if (newpos < 0 && !PyErr_Occurred())

0 commit comments

Comments
 (0)
0