From de17a828d543975215daf97b46df96e80774a414 Mon Sep 17 00:00:00 2001 From: Raj Date: Tue, 14 Feb 2023 14:29:24 +0530 Subject: [PATCH 01/16] gh-101892 : Updated Cython/objects/iterobject.c Line 217 Updated Cython/objects/iterobject.c Line 217 --- Objects/iterobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/iterobject.c b/Objects/iterobject.c index cfd6d0a7c959c9..736bb1efba5f47 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -214,7 +214,7 @@ calliter_iternext(calliterobject *it) } result = _PyObject_CallNoArgs(it->it_callable); - if (result != NULL) { + if (result != NULL && it->it_callable != NULL) { int ok; ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ); From 3fba831bf04687609be76f8d004ab018ee4c7aa8 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 09:08:50 +0000 Subject: [PATCH 02/16] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst diff --git a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst b/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst new file mode 100644 index 00000000000000..4176f95c06e530 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst @@ -0,0 +1 @@ +gh-101892 : Updated Cython/objects/iterobject.c Line 217 From bee2af77b15e42fc2c036ebae290b904359acd88 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 14 Feb 2023 13:13:36 +0400 Subject: [PATCH 03/16] Clarify the news entry --- .../next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst b/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst index 4176f95c06e530..be2baa0233b257 100644 --- a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst +++ b/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst @@ -1 +1 @@ -gh-101892 : Updated Cython/objects/iterobject.c Line 217 +Callable iterators no longer raise :class:`SystemError` by exhausting themselves. From 0c631e3413b0203cfb2335fdeec2363f3e8a31c4 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 14 Feb 2023 22:25:04 +0400 Subject: [PATCH 04/16] Add a test --- Lib/test/test_iter.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index acbdcb5f302060..82cd7eccc81835 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -267,6 +267,22 @@ def spam(state=[0]): return i self.check_iterator(iter(spam, 20), list(range(10)), pickle=False) + # Test two-argument iter() with function that empties its associated + # iterator with list() (or anything else but next()) then returns + # a non-sentinel value. + def test_iter_function_concealing_reentrant_exhaustion(self): + HAS_MORE = 1 + NO_MORE = 2 + def spam(): + if not spam.is_reentrant: + spam.is_reentrant = True + list(spam.iterator) + return HAS_MORE + return NO_MORE + spam.is_reentrant = False + spam.iterator = iter(spam, NO_MORE) + next(spam.iterator) + # Test exception propagation through function iterator def test_exception_function(self): def spam(state=[0]): From 506fb6ae34be7af0215a86a5a123901824562d81 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 14 Feb 2023 23:23:50 +0400 Subject: [PATCH 05/16] Amend wording --- Lib/test/test_iter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 82cd7eccc81835..50b979f1ea4db4 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -268,7 +268,7 @@ def spam(state=[0]): self.check_iterator(iter(spam, 20), list(range(10)), pickle=False) # Test two-argument iter() with function that empties its associated - # iterator with list() (or anything else but next()) then returns + # iterator via list() (or anything else but next()) then returns # a non-sentinel value. def test_iter_function_concealing_reentrant_exhaustion(self): HAS_MORE = 1 From 78168bf264a2c3784d95e8982cd01cc2385b914b Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 15 Feb 2023 09:07:43 +0400 Subject: [PATCH 06/16] Rep --- Lib/test/test_iter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 50b979f1ea4db4..869ca95d064b11 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -274,12 +274,12 @@ def test_iter_function_concealing_reentrant_exhaustion(self): HAS_MORE = 1 NO_MORE = 2 def spam(): - if not spam.is_reentrant: - spam.is_reentrant = True - list(spam.iterator) + if spam.need_reentrance: + spam.need_reentrance = False + list(spam.iterator) # Implicitly call ourselves return HAS_MORE return NO_MORE - spam.is_reentrant = False + spam.need_reentrance = True spam.iterator = iter(spam, NO_MORE) next(spam.iterator) From 188e388134f2126217a0f75aaa57e1398596135e Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 15 Feb 2023 11:06:46 +0400 Subject: [PATCH 07/16] Address the review --- Lib/test/test_iter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 869ca95d064b11..ecb95b7ed6b6ad 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -281,7 +281,8 @@ def spam(): return NO_MORE spam.need_reentrance = True spam.iterator = iter(spam, NO_MORE) - next(spam.iterator) + with self.assertRaises(StopIteration): + next(spam.iterator) # Test exception propagation through function iterator def test_exception_function(self): From 0f9cea13cddf037e535a5b08d330c5068cf68d11 Mon Sep 17 00:00:00 2001 From: Raj Date: Wed, 15 Feb 2023 12:45:09 +0530 Subject: [PATCH 08/16] gh-101892 : Updated Cython/objects/iterobject.c Line 217 --- Objects/iterobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 736bb1efba5f47..4340b0f58cc7c0 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -214,7 +214,7 @@ calliter_iternext(calliterobject *it) } result = _PyObject_CallNoArgs(it->it_callable); - if (result != NULL && it->it_callable != NULL) { + if (result != NULL && it->it_sentinel != NULL){ int ok; ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ); @@ -222,7 +222,6 @@ calliter_iternext(calliterobject *it) return result; /* Common case, fast path */ } - Py_DECREF(result); if (ok > 0) { Py_CLEAR(it->it_callable); Py_CLEAR(it->it_sentinel); @@ -233,6 +232,7 @@ calliter_iternext(calliterobject *it) Py_CLEAR(it->it_callable); Py_CLEAR(it->it_sentinel); } + Py_XDECREF(result); return NULL; } From c8b6bad05333cc2eeb9eeeef49c2dbfb17694b91 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 15 Feb 2023 11:52:49 +0400 Subject: [PATCH 09/16] Update test_iter.py --- Lib/test/test_iter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index ecb95b7ed6b6ad..a7bcf3681c5cd9 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -268,8 +268,8 @@ def spam(state=[0]): self.check_iterator(iter(spam, 20), list(range(10)), pickle=False) # Test two-argument iter() with function that empties its associated - # iterator via list() (or anything else but next()) then returns - # a non-sentinel value. + # iterator via list() (or anything else but next()) but returns + # a non-sentinel value thus claiming that the iterator can yield more. def test_iter_function_concealing_reentrant_exhaustion(self): HAS_MORE = 1 NO_MORE = 2 From 6931c20c42ae9d9bb139d2e1d192a47cda8add14 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 15 Feb 2023 11:57:50 +0400 Subject: [PATCH 10/16] Remove an unnecessary abstraction of reentrance --- Lib/test/test_iter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index a7bcf3681c5cd9..efec073a7ff187 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -274,12 +274,12 @@ def test_iter_function_concealing_reentrant_exhaustion(self): HAS_MORE = 1 NO_MORE = 2 def spam(): - if spam.need_reentrance: - spam.need_reentrance = False + if spam.not_emptied: + spam.not_emptied = False list(spam.iterator) # Implicitly call ourselves return HAS_MORE return NO_MORE - spam.need_reentrance = True + spam.not_emptied = True spam.iterator = iter(spam, NO_MORE) with self.assertRaises(StopIteration): next(spam.iterator) From 36b043ccc642f64508bf98b2f5c269c0aaf3f75b Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 15 Feb 2023 22:47:47 +0400 Subject: [PATCH 11/16] Clarify comments once more --- Lib/test/test_iter.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index efec073a7ff187..0b838fa2fd012a 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -91,6 +91,12 @@ def __call__(self): raise IndexError # Emergency stop return i + +def exhaust(iterator): + """Exhaust an iterator without raising StopIteration.""" + list(iterator) + + # Main test suite class TestCase(unittest.TestCase): @@ -267,19 +273,22 @@ def spam(state=[0]): return i self.check_iterator(iter(spam, 20), list(range(10)), pickle=False) - # Test two-argument iter() with function that empties its associated - # iterator via list() (or anything else but next()) but returns - # a non-sentinel value thus claiming that the iterator can yield more. + # Test two-argument iter() with function that exhausts its + # associated iterator but forgets to either return a sentinel value + # or raise `StopIteration`. def test_iter_function_concealing_reentrant_exhaustion(self): HAS_MORE = 1 NO_MORE = 2 def spam(): - if spam.not_emptied: - spam.not_emptied = False - list(spam.iterator) # Implicitly call ourselves - return HAS_MORE - return NO_MORE - spam.not_emptied = True + # Touching the iterator with exhaust() below will call + # spam() once again + if spam.is_recursive_call: + return NO_MORE + spam.is_recursive_call = True + exhaust(spam.iterator) + return HAS_MORE + + spam.is_recursive_call = False spam.iterator = iter(spam, NO_MORE) with self.assertRaises(StopIteration): next(spam.iterator) From 340f668f6f6b35646432b33ca292565a3cc8fc17 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 15 Feb 2023 22:52:01 +0400 Subject: [PATCH 12/16] Update 2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst --- .../Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst b/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst index be2baa0233b257..d586779b3a8a36 100644 --- a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst +++ b/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst @@ -1 +1,3 @@ -Callable iterators no longer raise :class:`SystemError` by exhausting themselves. +Callable iterators no longer raise :class:`SystemError` when the +callable object exhausts the iterator but forgets to either return a +sentinel value or raise :class:`StopIteration`. From fe1502dab4d2c60a68d413138e0d4a55a42c8222 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 15 Feb 2023 23:16:35 +0400 Subject: [PATCH 13/16] Fix issues reported by patchcheck --- Lib/test/test_iter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 0b838fa2fd012a..2f887122ac7f4e 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -94,7 +94,7 @@ def __call__(self): def exhaust(iterator): """Exhaust an iterator without raising StopIteration.""" - list(iterator) + list(iterator) # Main test suite @@ -273,7 +273,7 @@ def spam(state=[0]): return i self.check_iterator(iter(spam, 20), list(range(10)), pickle=False) - # Test two-argument iter() with function that exhausts its + # Test two-argument iter() with function that exhausts its # associated iterator but forgets to either return a sentinel value # or raise `StopIteration`. def test_iter_function_concealing_reentrant_exhaustion(self): @@ -287,7 +287,7 @@ def spam(): spam.is_recursive_call = True exhaust(spam.iterator) return HAS_MORE - + spam.is_recursive_call = False spam.iterator = iter(spam, NO_MORE) with self.assertRaises(StopIteration): From cc2066b0224ab6cb8cb78c0ebf8495b8709f42ee Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sat, 4 Mar 2023 13:36:35 +0400 Subject: [PATCH 14/16] Do not nitpick for empty lines of others --- Lib/test/test_iter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 208e6ab6cfaeaf..cc1046bc6fc4cd 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -106,7 +106,6 @@ def __len__(self): def __getitem__(self, i): raise StopIteration - # Main test suite class TestCase(unittest.TestCase): From 27e478f657643bba8388f40fa49d324bfa2b5050 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sat, 4 Mar 2023 13:37:51 +0400 Subject: [PATCH 15/16] Fix mismerging: move the new method --- Lib/test/test_iter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index cc1046bc6fc4cd..11bac515733813 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -94,18 +94,18 @@ def __call__(self): raise IndexError # Emergency stop return i - -def exhaust(iterator): - """Exhaust an iterator without raising StopIteration.""" - list(iterator) - - class EmptyIterClass: def __len__(self): return 0 def __getitem__(self, i): raise StopIteration + +def exhaust(iterator): + """Exhaust an iterator without raising StopIteration.""" + list(iterator) + + # Main test suite class TestCase(unittest.TestCase): From 1ee22939750381386bc414d187f5011635700290 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sat, 4 Mar 2023 14:28:43 +0400 Subject: [PATCH 16/16] Address the review --- Lib/test/test_iter.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 11bac515733813..30aedb0db3bb3d 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -100,12 +100,6 @@ def __len__(self): def __getitem__(self, i): raise StopIteration - -def exhaust(iterator): - """Exhaust an iterator without raising StopIteration.""" - list(iterator) - - # Main test suite class TestCase(unittest.TestCase): @@ -354,15 +348,20 @@ def spam(state=[0]): return i self.check_iterator(iter(spam, 20), list(range(10)), pickle=False) - # Test two-argument iter() with function that exhausts its - # associated iterator but forgets to either return a sentinel value - # or raise `StopIteration`. def test_iter_function_concealing_reentrant_exhaustion(self): + # gh-101892: Test two-argument iter() with a function that + # exhausts its associated iterator but forgets to either return + # a sentinel value or raise StopIteration. HAS_MORE = 1 NO_MORE = 2 + + def exhaust(iterator): + """Exhaust an iterator without raising StopIteration.""" + list(iterator) + def spam(): # Touching the iterator with exhaust() below will call - # spam() once again + # spam() once again so protect against recursion. if spam.is_recursive_call: return NO_MORE spam.is_recursive_call = True