8000 bpo-43260: io: Prevent large data remains in textio buffer. (GH-24592) · python/cpython@6e2f144 · GitHub
[go: up one dir, main page]
Skip to content

Commit 6e2f144

Browse files
methanemethane
authored
bpo-43260: io: Prevent large data remains in textio buffer. (GH-24592)
When very large data remains in TextIOWrapper, flush() may fail forever. So prevent that data larger than chunk_size is remained in TextIOWrapper internal buffer. Co-Authored-By: Eryk Sun. (cherry picked from commit 01806d5)
1 parent 6ddb255 commit 6e2f144

File tree

3 files changed

+46
-3
lines changed

3 files changed

+46
-3
lines changed

‎Lib/test/test_io.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3736,6 +3736,33 @@ def test_del__CHUNK_SIZE_SystemError(self):
37363736
with self.assertRaises(AttributeError):
37373737
del t._CHUNK_SIZE
37383738

3739+
def test_internal_buffer_size(self):
3740+
# bpo-43260: TextIOWrapper's internal buffer should not store
3741+
# data larger than chunk size.
3742+
chunk_size = 8192 # default chunk size, updated later
3743+
3744+
class MockIO(self.MockRawIO):
3745+
def write(self, data):
3746+
if len(data) > chunk_size:
3747+
raise RuntimeError
3748+
return super().write(data)
3749+
3750+
buf = MockIO()
3751+
t = self.TextIOWrapper(buf, encoding="ascii")
3752+
chunk_size = t._CHUNK_SIZE
3753+
t.write("abc")
3754+
t.write("def")
3755+
# default chunk size is 8192 bytes so t don't write data to buf.
3756+
self.assertEqual([], buf._write_stack< 8000 /span>)
3757+
3758+
with self.assertRaises(RuntimeError):
3759+
t.write("x"*(chunk_size+1))
3760+
3761+
self.assertEqual([b"abcdef"], buf._write_stack)
3762+
t.write("ghi")
3763+
t.write("x"*chunk_size)
3764+
self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack)
3765+
37393766

37403767
class PyTextIOWrapperTest(TextIOWrapperTest):
37413768
io = pyio

‎Misc/NEWS.d/next/Library/2021-02-20-12-15-29.bpo-43260.6znAas.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix TextIOWrapper can not flush internal buffer forever after very large
2+
text is written.

‎Modules/_io/textio.c

Lines changed: 17 additions & 3 deletions
< 8000 /tr>
Original file line numberDiff line numberDiff line change
@@ -1558,6 +1558,8 @@ _textiowrapper_writeflush(textio *self)
15581558
_PyIO_str_write, b, NULL);
15591559
} while (ret == NULL && _PyIO_trap_eintr());
15601560
Py_DECREF(b);
1561+
// NOTE: We cleared buffer but we don't know how many bytes are actually written
1562+
// when an error occurred.
15611563
if (ret == NULL)
15621564
return -1;
15631565
Py_DECREF(ret);
@@ -1615,7 +1617,10 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text)
16151617

16161618
/* XXX What if we were just reading? */
16171619
if (self->encodefunc != NULL) {
1618-
if (PyUnicode_IS_ASCII(text) && is_asciicompat_encoding(self->encodefunc)) {
1620+
if (PyUnicode_IS_ASCII(text) &&
1621+
// See bpo-43260
1622+
PyUnicode_GET_LENGTH(text) <= self->chunk_size &&
1623+
is_asciicompat_encoding(self->encodefunc)) {
16191624
b = text;
16201625
Py_INCREF(b);
16211626
}
@@ -1624,9 +1629,10 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text)
16241629
}
16251630
self->encoding_start_of_stream = 0;
16261631
}
1627-
else
1632+
else {
16281633
b = PyObject_CallMethodObjArgs(self->encoder,
16291634
_PyIO_str_encode, text, NULL);
1635+
}
16301636

16311637
Py_DECREF(text);
16321638
if (b == NULL)
@@ -1651,6 +1657,14 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text)
16511657
self->pending_bytes_count = 0;
16521658
self->pending_bytes = b;
16531659
}
1660+
else if (self->pending_bytes_count + bytes_len > self->chunk_size) {
1661+
// Prevent to concatenate more than chunk_size data.
1662+
if (_textiowrapper_writeflush(self) < 0) {
1663+
Py_DECREF(b);
1664+
return NULL;
1665+
}
1666+
self->pending_bytes = b;
1667+
}
16541668
else if (!PyList_CheckExact(self->pending_bytes)) {
16551669
PyObject *list = PyList_New(2);
16561670
if (list == NULL) {
@@ -1670,7 +1684,7 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text)
16701684
}
16711685

16721686
self->pending_bytes_count += bytes_len;
1673-
if (self->pending_bytes_count > self->chunk_size || needflush ||
1687+
if (self->pending_bytes_count >= self->chunk_size || needflush ||
16741688
text_needflush) {
16751689
if (_textiowrapper_writeflush(self) < 0)
16761690
return NULL;

0 commit comments

Comments
 (0)
0