8000 gh-70278: Fix PyUnicode_FromFormat() with precision for %s and %V by serhiy-storchaka · Pull Request #120365 · python/cpython · GitHub
[go: up one dir, main page]
Skip to content

gh-70278: Fix PyUnicode_FromFormat() with precision for %s and %V #120365

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jun 24, 2024
Merged

gh-70278: Fix PyUnicode_FromFormat() with precision for %s and %V #120365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e969269
gh-70278: Fix PyUnicode_FromFormat() with precision for %s and %V
serhiy-storchaka Jun 11, 2024
6f80f9c
Fix a case when the precision is larger than strlen().
serhiy-storchaka Jun 11, 2024
fdc6d4d
Merge branch 'main' into PyUnicode_FromFormat-truncate
serhiy-storchaka Jun 12, 2024
4000cdf
Add a comment and refactor.
serhiy-storchaka Jun 12, 2024
c628994
Merge branch 'main' into PyUnicode_FromFormat-truncate
serhiy-storchaka Jun 24, 2024
928202a
Merge remote-tracking branch 'refs/remotes/origin/PyUnicode_FromForma…
serhiy-storchaka Jun 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 44 additions & 2 deletions Lib/test/test_capi/test_unicode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -419,8 +419,29 @@ def check_format(expected, format, *args):
# truncated string
check_format('abc',
b'%.3s', b'abcdef')
check_format('abc[',
b'%.6s', 'abc[\u20ac]'.encode('utf8'))
check_format('abc[\u20ac',
b'%.7s', 'abc[\u20ac]'.encode('utf8'))
check_format('abc[\ufffd',
b'%.5s', 'abc[\u20ac]'.encode('utf8'))
b'%.5s', b'abc[\xff]')
check_format('abc[',
b'%.6s', b'abc[\xe2\x82]')
check_format('abc[\ufffd]',
b'%.7s', b'abc[\xe2\x82]')
check_format('abc[\ufffd',
b'%.7s', b'abc[\xe2\x82\0')
check_format(' abc[',
b'%10.6s', 'abc[\u20ac]'.encode('utf8'))
check_format(' abc[\u20ac',
b'%10.7s', 'abc[\u20ac]'.encode('utf8'))
check_format(' abc[\ufffd',
b'%10.5s', b'abc[\xff]')
check_format(' abc[',
b'%10.6s', b'abc[\xe2\x82]')
check_format(' abc[\ufffd]',
b'%10.7s', b'abc[\xe2\x82]')

check_format("'\\u20acABC'",
b'%A', '\u20acABC')
check_format("'\\u20",
Expand All @@ -433,10 +454,31 @@ def check_format(expected, format, *args):
b'%.3S', '\u20acABCDEF')
check_format('\u20acAB',
b'%.3U', '\u20acABCDEF')

check_format('\u20acAB',
b'%.3V', '\u20acABCDEF', None)
check_format('abc[',
b'%.6V', None, 'abc[\u20ac]'.encode('utf8'))
check_format('abc[\u20ac',
b'%.7V', None, 'abc[\u20ac]'.encode('utf8'))
check_format('abc[\ufffd',
b'%.5V', None, 'abc[\u20ac]'.encode('utf8'))
b'%.5V', None, b'abc[\xff]')
check_format('abc[',
b'%.6V', None, b'abc[\xe2\x82]')
check_format('abc[\ufffd]',
b'%.7V', None, b'abc[\xe2\x82]')
check_format(' abc[',
b'%10.6V', None, 'abc[\u20ac]'.encode('utf8'))
check_format(' abc[\u20ac',
b'%10.7V', None, 'abc[\u20ac]'.encode('utf8'))
check_format(' abc[\ufffd',
b'%10.5V', None, b'abc[\xff]')
check_format(' abc[',
b'%10.6V', None, b'abc[\xe2\x82]')
check_format(' abc[\ufffd]',
b'%10.7V', None, b'abc[\xe2\x82]')
check_format(' abc[\ufffd',
b'%10.7V', None, b'abc[\xe2\x82\0')

# following tests comes from #7330
# test width modifier and precision modifier with %S
Expand Down
4 changes: 4 additions & 0 deletions Misc/NEWS.d/next/C API/2024-06-11-21-38-32.gh-issue-70278.WDE4zM.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
:c:func:`PyUnicode_FromFormat` no longer produces the ending ``\ufffd``
character for truncated C string when use precision with ``%s`` and ``%V``.
It now truncates the string before the start of truncated multibyte
sequences.
13 changes: 11 additions & 2 deletions Objects/unicodeobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2581,6 +2581,7 @@ unicode_fromformat_write_utf8(_PyUnicodeWriter *writer, const char *str,
Py_ssize_t width, Py_ssize_t precision, int flags)
{
/* UTF-8 */
Py_ssize_t *pconsumed = NULL;
Py_ssize_t length;
if (precision == -1) {
length = strlen(str);
Expand All @@ -2590,15 +2591,23 @@ unicode_fromformat_write_utf8(_PyUnicodeWriter *writer, const char *str,
while (length < precision && str[length]) {
length++;
}
if (length == precision) {
/* The input string is not NUL-terminated. If it ends with an
* incomplete UTF-8 sequence, truncate the string just before it.
* Incomplete sequences in the middle and sequences which cannot
* be valid prefixes are still treated as errors and replaced
* with \xfffd. */
pconsumed = &length;
}
}

if (width < 0) {
return unicode_decode_utf8_writer(writer, str, length,
_Py_ERROR_REPLACE, "replace", NULL);
_Py_ERROR_REPLACE, "replace", pconsumed);
}

PyObject *unicode = PyUnicode_DecodeUTF8Stateful(str, length,
"replace", NULL);
"replace", pconsumed);
if (unicode == NULL)
return -1;

Expand Down
Loading
0