8000 Issue #23215: Multibyte codecs with custom error handlers that ignore… · python/cpython@df80706 · GitHub
[go: up one dir, main page]
Skip to content

Commit df80706

Browse files
serhiy-storchakaserhiy-storchaka
committed
Issue #23215: Multibyte codecs with custom error handlers that ignores errors
consumed too much memory and raised SystemError or MemoryError. Original patch by Aleksi Torhamo.
2 parents 254dd59 + a1543cd commit df80706

File tree

3 files changed

+22
-8
lines changed

3 files changed

+22
-8
lines changed

‎Lib/test/test_multibytecodec.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@ def test_errorcallback_longindex(self):
4444
self.assertRaises(IndexError, dec,
4545
b'apple\x92ham\x93spam', 'test.cjktest')
4646

47+
def test_errorcallback_custom_ignore(self):
48+
# Issue #23215: MemoryError with custom error handlers and multibyte codecs
49+
data = 100 * "\udc00"
50+
codecs.register_error("test.ignore", codecs.ignore_errors)
51+
for enc in ALL_CJKENCODINGS:
52+
self.assertEqual(data.encode(enc, "test.ignore"), b'')
53+
4754
def test_codingspec(self):
4855
try:
4956
for enc in ALL_CJKENCODINGS:

‎Misc/NEWS

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ Core and Builtins
1313
Library
1414
-------
1515

16+
- Issue #23215: Multibyte codecs with custom error handlers that ignores errors
17+
consumed too much memory and raised SystemError or MemoryError.
18+
Original patch by Aleksi Torhamo.
19+
1620
- Issue #5700: io.FileIO() called flush() after closing the file.
1721
flush() was not called in close() if closefd=False.
1822

‎Modules/cjkcodecs/multibytecodec.c

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -174,8 +174,10 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
174174
orgsize = PyBytes_GET_SIZE(buf->outobj);
175175
incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);
176176

177-
if (orgsize > PY_SSIZE_T_MAX - incsize)
177+
if (orgsize > PY_SSIZE_T_MAX - incsize) {
178+
PyErr_NoMemory();
178179
return -1;
180+
}
179181

180182
if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1)
181183
return -1;
@@ -186,11 +188,11 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
186188

187189
return 0;
188190
}
189-
#define REQUIRE_ENCODEBUFFER(buf, s) { \
190-
if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \
191+
#define REQUIRE_ENCODEBUFFER(buf, s) do { \
192+
if ((s) < 0 || (s) > (buf)->outb BB71 uf_end - (buf)->outbuf) \
191193
if (expand_encodebuffer(buf, s) == -1) \
192194
goto errorexit; \
193-
}
195+
} while(0)
194196

195197

196198
/**
@@ -324,10 +326,11 @@ multibytecodec_encerror(MultibyteCodec *codec,
324326

325327
assert(PyBytes_Check(retstr));
326328
retstrsize = PyBytes_GET_SIZE(retstr);
327-
REQUIRE_ENCODEBUFFER(buf, retstrsize);
328-
329-
memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize);
330-
buf->outbuf += retstrsize;
329+
if (retstrsize > 0) {
330+
REQUIRE_ENCODEBUFFER(buf, retstrsize);
331+
memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize);
332+
buf->outbuf += retstrsize;
333+
}
331334

332335
newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
333336
if (newpos < 0 && !PyErr_Occurred())

0 commit comments

Comments
 (0)
0