8000 [3.11] gh-119506: fix _io.TextIOWrapper.write() write during flush (#… · python/cpython@8a978a7 · GitHub
[go: up one dir, main page]
Skip to content

Commit 8a978a7

Browse files
methanehugovk
methane
and
hugovk
authored
[3.11] gh-119506: fix _io.TextIOWrapper.write() write during flush (#119507) (#120314)
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> fix _io.TextIOWrapper.write() write during flush (#119507)
1 parent b396360 commit 8a978a7

File tree

3 files changed

+67
-22
lines changed

3 files changed

+67
-22
lines changed

‎Lib/test/test_io.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3960,6 +3960,28 @@ def write(self, data):
39603960
t.write("x"*chunk_size)
39613961
self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack)
39623962

3963+
def test_issue119506(self):
3964+
chunk_size = 8192
3965+
3966+
class MockIO(self.MockRawIO):
3967+
written = False
3968+
def write(self, data):
3969+
if not self.written:
3970+
self.written = True
3971+
t.write("middle")
3972+
return super().write(data)
3973+
3974+
buf = MockIO()
3975+
t = self.TextIOWrapper(buf)
3976+
t.write("abc")
3977+
t.write("def")
3978+
# writing data which size >= chunk_size cause flushing buffer before write.
3979+
t.write("g" * chunk_size)
3980+
t.flush()
3981+
3982+
self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size],
3983+
buf._write_stack)
3984+
39633985

39643986
class PyTextIOWrapperTest(TextIOWrapperTest):
39653987
io = pyio

‎Misc/NEWS.d/next/Library/2024-05-24-14-32-24.gh-issue-119506.-nMNqq.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix :meth:`!io.TextIOWrapper.write` method breaks internal buffer when the method is called again during flushing internal buffer.

‎Modules/_io/textio.c

Lines changed: 44 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1701,34 +1701,56 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text)
17011701
bytes_len = PyBytes_GET_SIZE(b);
17021702
}
17031703

1704-
if (self->pending_bytes == NULL) {
1705-
self->pending_bytes_count = 0;
1706-
self->pending_bytes = b;
1707-
}
1708-
else if (self->pending_bytes_count + bytes_len > self->chunk_size) {
1709-
// Prevent to concatenate more than chunk_size data.
1710-
if (_textiowrapper_writeflush(self) < 0) {
1711-
Py_DECREF(b);
1712-
return NULL;
1704+
// We should avoid concatinating huge data.
1705+
// Flush the buffer before adding b to the buffer if b is not small.
1706+
// https://github.com/python/cpython/issues/87426
1707+
if (bytes_len >= self->chunk_size) {
1708+
// _textiowrapper_writeflush() calls buffer.write().
1709+
// self->pending_bytes can be appended during buffer->write()
1710+
// or other thread.
1711+
// We need to loop until buffer becomes empty.
1712+
// https://github.com/python/cpython/issues/118138
1713+
// https://github.com/python/cpython/issues/119506
1714+
while (self->pending_bytes != NULL) {
1715+
if (_textiowrapper_writeflush(self) < 0) {
1716+
Py_DECREF(b);
1717+
return NULL;
1718+
}
17131719
}
1714-
self->pending_bytes = b;
17151720
}
1716-
else if (!PyList_CheckExact(self->pending_bytes)) {
1717-
PyObject *list = PyList_New(2);
1718-
if (list == NULL) {
1719-
Py_DECREF(b);
1720-
return NULL;
1721-
}
1722-
PyList_SET_ITEM(list, 0, self->pending_bytes);
1723-
PyList_SET_ITEM(list, 1, b);
1724-
self->pending_bytes = list;
1721+
1722+
if (self->pending_bytes == NULL) {
1723+
assert(self->pending_bytes_count == 0);
1724+
self->pending_bytes = b;
17251725
}
17261726
else {
1727-
if (PyList_Append(self->pending_bytes, b) < 0) {
1728-
Py_DECREF(b);
1729-
return NULL;
1727+
if (!PyList_CheckExact(self->pending_bytes)) {
1728+
PyObject *list = PyList_New(0);
1729+
if (list == NULL) {
1730+
Py_DECREF(b);
1731+
return NULL;
1732+
}
1733+
// PyList_New() may trigger GC and other thread may call write().
1734+
// So, we need to check the self->pending_bytes is a list again.
1735+
if (PyList_CheckExact(self->pending_bytes)) {
1736+
// Releasing empty list won't trigger GC and/or __del__.
1737+
Py_DECREF(list);
1738+
}
1739+
else {
1740+
if (PyList_Append(list, self->pending_bytes) < 0) {
1741+
Py_DECREF(list);
1742+
Py_DECREF(b);
1743+
return NULL;
1744+
}
1745+
Py_SETREF(self->pending_bytes, list);
1746+
}
17301747
}
1748+
1749+
int ret = PyList_Append(self->pending_bytes, b);
17311750
Py_DECREF(b);
1751+
if (ret < 0) {
1752+
return NULL;
1753+
}
17321754
}
17331755

17341756
self->pending_bytes_count += bytes_len;

0 commit comments

Comments
 (0)
0