8000 ENH: skip or avoid gc/objectmodel differences btwn pypy and cpython by mattip · Pull Request #7912 · numpy/numpy · GitHub
[go: up one dir, main page]
Skip to content

ENH: skip or avoid gc/objectmodel differences btwn pypy and cpython #7912

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 15, 2016
Merged

ENH: skip or avoid gc/objectmodel differences btwn pypy and cpython #7912

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
8000
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
ENH: skip or avoid gc/objectmodel differences btwn pypy and cpython
  • Loading branch information
@mattip
mattip committed Aug 15, 2016
commit 76eff9ca4648e96ba1f560e83f40383f68805493
23 changes: 13 additions & 10 deletions numpy/core/tests/test_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from numpy.compat import sixu
from numpy.testing import (
run_module_suite, assert_, assert_equal, assert_array_equal,
assert_raises
assert_raises, HAS_REFCOUNT
)

# Switch between new behaviour when NPY_RELAXED_STRIDES_CHECKING is set.
Expand All @@ -19,23 +19,26 @@ def test_array_array():
tndarray = type(ones11)
# Test is_ndarray
assert_equal(np.array(ones11, dtype=np.float64), ones11)
old_refcount = sys.getrefcount(tndarray)
np.array(ones11)
assert_equal(old_refcount, sys.getrefcount(tndarray))
if HAS_REFCOUNT:
old_refcount = sys.getrefcount(tndarray)
np.array(ones11)
assert_equal(old_refcount, sys.getrefcount(tndarray))

# test None
assert_equal(np.array(None, dtype=np.float64),
np.array(np.nan, dtype=np.float64))
old_refcount = sys.getrefcount(tobj)
np.array(None, dtype=np.float64)
assert_equal(old_refcount, sys.getrefcount(tobj))
if HAS_REFCOUNT:
old_refcount = sys.getrefcount(tobj)
np.array(None, dtype=np.float64)
assert_equal(old_refcount, sys.getrefcount(tobj))

# test scalar
assert_equal(np.array(1.0, dtype=np.float64),
np.ones((), dtype=np.float64))
old_refcount = sys.getrefcount(np.float64)
np.array(np.array(1.0, dtype=np.float64), dtype=np.float64)
assert_equal(old_refcount, sys.getrefcount(np.float64))
if HAS_REFCOUNT:
old_refcount = sys.getrefcount(np.float64)
np.array(np.array(1.0, dtype=np.float64), dtype=np.float64)
assert_equal(old_refcount, sys.getrefcount(np.float64))

# test string
S2 = np.dtype((str, 2))
Expand Down
31 changes: 19 additions & 12 deletions numpy/core/tests/test_indexing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from itertools import product
from numpy.testing import (
TestCase, run_module_suite, assert_, assert_equal, assert_raises,
assert_array_equal, assert_warns
assert_array_equal, assert_warns, HAS_REFCOUNT
)


Expand Down Expand Up @@ -411,7 +411,8 @@ def __array__(self):
def test_small_regressions(self):
# Reference count of intp for index checks
a = np.array([0])
refcount = sys.getrefcount(np.dtype(np.intp))
if HAS_REFCOUNT:
refcount = sys.getrefcount(np.dtype(np.intp))
# item setting always checks indices in separate function:
a[np.array([0], dtype=np.intp)] = 1
a[np.array([0], dtype=np.uint8)] = 1
Expand All @@ -420,7 +421,8 @@ def test_small_regressions(self):
assert_raises(IndexError, a.__setitem__,
np.array([1], dtype=np.uint8), 1)

assert_equal(sys.getrefcount(np.dtype(np.intp)), refcount)
if HAS_REFCOUNT:
assert_equal(sys.getrefcount(np.dtype(np.intp)), refcount)

def test_unaligned(self):
v = (np.zeros(64, dtype=np.int8) + ord('a'))[1:-7]
Expand Down Expand Up @@ -972,10 +974,12 @@ def _check_multi_index(self, arr, index):
try:
mimic_get, no_copy = self._get_multi_index(arr, index)
except Exception:
prev_refcount = sys.getrefcount(arr)
if HAS_REFCOUNT:
prev_refcount = sys.getrefcount(arr)
assert_raises(Exception, arr.__getitem__, index)
assert_raises(Exception, arr.__setitem__, index, 0)
assert_equal(prev_refcount, sys.getrefcount(arr))
if HAS_REFCOUNT:
assert_equal(prev_refcount, sys.getrefcount(arr))
return

self._compare_index_result(arr, index, mimic_get, no_copy)
Expand All @@ -994,10 +998,12 @@ def _check_single_index(self, arr, index):
try:
mimic_get, no_copy = self._get_multi_index(arr, (index,))
except Exception:
prev_refcount = sys.getrefcount(arr)
if HAS_REFCOUNT:
prev_refcount = sys.getrefcount(arr)
assert_raises(Exception, arr.__getitem__, index)
assert_raises(Exception, arr.__setitem__, index, 0)
assert_equal(prev_refcount, sys.getrefcount(arr))
if HAS_REFCOUNT:
assert_equal(prev_refcount, sys.getrefcount(arr))
return

self._compare_index_result(arr, index, mimic_get, no_copy)
Expand All @@ -1013,11 +1019,12 @@ def _compare_index_result(self, arr, index, mimic_get, no_copy):
if indexed_arr.size != 0 and indexed_arr.ndim != 0:
assert_(np.may_share_memory(indexed_arr, arr) == no_copy)
# Check reference count of the original array
if no_copy:
# refcount increases by one:
assert_equal(sys.getrefcount(arr), 3)
else:
assert_equal(sys.getrefcount(arr), 2)
if HAS_REFCOUNT:
if no_copy:
# refcount increases by one:
assert_equal(sys.getrefcount(arr), 3)
else:
assert_equal(sys.getrefcount(arr), 2)

# Test non-broadcast setitem:
b = arr.copy()
Expand Down
8 changes: 5 additions & 3 deletions numpy/core/tests/test_item_selection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import numpy as np
from numpy.testing import (
TestCase, run_module_suite, assert_, assert_raises,
assert_array_equal
assert_array_equal, HAS_REFCOUNT
)


Expand Down Expand Up @@ -55,12 +55,14 @@ def test_refcounting(self):
b = np.array([2, 2, 4, 5, 3, 5])
a.take(b, out=a[:6])
del a
assert_(all(sys.getrefcount(o) == 3 for o in objects))
if HAS_REFCOUNT:
assert_(all(sys.getrefcount(o) == 3 for o in objects))
# not contiguous, example:
a = np.array(objects * 2)[::2]
a.take(b, out=a[:6])
del a
assert_(all(sys.getrefcount(o) == 3 for o in objects))
if HAS_REFCOUNT:
assert_(all(sys.getrefcount(o) == 3 for o in objects))

def test_unicode_mode(self):
d = np.arange(10)
Expand Down
85 changes: 46 additions & 39 deletions numpy/core/tests/test_multiarray.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
from numpy.testing import (
TestCase, run_module_suite, assert_, assert_raises,
assert_equal, assert_almost_equal, assert_array_equal,
assert_array_almost_equal, assert_allclose,
assert_array_almost_equal, assert_allclose, IS_PYPY, HAS_REFCOUNT,
assert_array_less, runstring, dec, SkipTest, temppath
)

Expand Down Expand Up @@ -2326,10 +2326,12 @@ def test_diagonal_view_notwriteable(self):
def test_diagonal_memleak(self):
# Regression test for a bug that crept in at one point
a = np.zeros((100, 100))
assert_(sys.getrefcount(a) < 50)
if HAS_REFCOUNT:
assert_(sys.getrefcount(a) < 50)
for i in range(100):
a.diagonal()
assert_(sys.getrefcount(a) < 50)
if HAS_REFCOUNT:
assert_(sys.getrefcount(a) < 50)

def test_trace(self):
a = np.arange(12).reshape((3, 4))
Expand Down Expand Up @@ -4723,7 +4725,8 @@ def test_dot_3args(self):
r = np.empty((1024, 32))
for i in range(12):
dot(f, v, r)
assert_equal(sys.getrefcount(r), 2)
if HAS_REFCOUNT:
assert_equal(sys.getrefcount(r), 2)
r2 = dot(f, v, out=None)
assert_array_equal(r2, r)
assert_(r is dot(f, v, out=r))
Expand Down Expand Up @@ -5967,11 +5970,13 @@ def test_padding(self):
self._check_roundtrip(x)

def test_reference_leak(self):
count_1 = sys.getrefcount(np.core._internal)
if HAS_REFCOUNT:
count_1 = sys.getrefcount(np.core._internal)
a = np.zeros(4)
b = memoryview(a)
c = np.asarray(b)
count_2 = sys.getrefcount(np.core._internal)
if HAS_REFCOUNT:
count_2 = sys.getrefcount(np.core._internal)
assert_equal(count_1, count_2)
del c # avoid pyflakes unused variable warning.

Expand Down Expand Up @@ -6355,50 +6360,52 @@ def test_string(self):
assert_equal(np.where(False, b, a), "abcd")


class TestSizeOf(TestCase):
if not IS_PYPY:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another way to do this in Nose is probably to raise SkipTest in def setup(self): ...

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting suggestion. Probably not as obvious in reading the code, but otherwise cleaner, at least until we drop the now abandoned Nose.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using dec.skip() in a setup() class method did not skip the class. It seems to skip one or more tests but not all. I could not find documentation or examples of how to do this in the 10 minutes I tried.

# sys.getsizeof() is not valid on PyPy
class TestSizeOf(TestCase):

def test_empty_array(self):
x = np.array([])
assert_(sys.getsizeof(x) > 0)
def test_empty_array(self):
x = np.array([])
assert_(sys.getsizeof(x) > 0)

def check_array(self, dtype):
elem_size = dtype(0).itemsize
def check_array(self, dtype):
elem_size = dtype(0).itemsize

for length in [10, 50, 100, 500]:
x = np.arange(length, dtype=dtype)
assert_(sys.getsizeof(x) > length * elem_size)
for length in [10, 50, 100, 500]:
x = np.arange(length, dtype=dtype)
assert_(sys.getsizeof(x) > length * elem_size)

def test_array_int32(self):
self.check_array(np.int32)
def test_array_int32(self):
self.check_array(np.int32)

def test_array_int64(self):
self.check_array(np.int64)
def test_array_int64(self):
self.check_array(np.int64)

def test_array_float32(self):
self.check_array(np.float32)
def test_array_float32(self):
self.check_array(np.float32)

def test_array_float64(self):
self.check_array(np.float64)
def test_array_float64(self):
self.check_array(np.float64)

def test_view(self):
d = np.ones(100)
assert_(sys.getsizeof(d[...]) < sys.getsizeof(d))
def test_view(self):
d = np.ones(100)
assert_(sys.getsizeof(d[...]) < sys.getsizeof(d))

def test_reshape(self):
d = np.ones(100)
assert_(sys.getsizeof(d) < sys.getsizeof(d.reshape(100, 1, 1).copy()))
def test_reshape(self):
d = np.ones(100)
assert_(sys.getsizeof(d) < sys.getsizeof(d.reshape(100, 1, 1).copy()))

def test_resize(self):
d = np.ones(100)
old = sys.getsizeof(d)
d.resize(50)
assert_(old > sys.getsizeof(d))
d.resize(150)
assert_(old < sys.getsizeof(d))
def test_resize(self):
d = np.ones(100)
old = sys.getsizeof(d)
d.resize(50)
assert_(old > sys.getsizeof(d))
d.resize(150)
assert_(old < sys.getsizeof(d))

def test_error(self):
d = np.ones(100)
assert_raises(TypeError, d.__sizeof__, "a")
def test_error(self):
d = np.ones(100)
assert_raises(TypeError, d.__sizeof__, "a")


class TestHashing(TestCase):
Expand Down
30 changes: 20 additions & 10 deletions numpy/core/tests/test_nditer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from numpy.core.multiarray_tests import test_nditer_too_large
from numpy.testing import (
run_module_suite, assert_, assert_equal, assert_array_equal,
assert_raises, dec
assert_raises, dec, HAS_REFCOUNT
)


Expand All @@ -34,6 +34,7 @@ def iter_iterindices(i):
i.iternext()
return ret

@dec.skipif(not HAS_REFCOUNT, "python does not have sys.getrefcount")
def test_iter_refcount():
# Make sure the iterator doesn't leak

Expand Down Expand Up @@ -1000,32 +1001,37 @@ def test_iter_object_arrays_basic():

obj = {'a':3,'b':'d'}
a = np.array([[1, 2, 3], None, obj, None], dtype='O')
rc = sys.getrefcount(obj)
if HAS_REFCOUNT:
rc = sys.getrefcount(obj)

# Need to allow references for object arrays
assert_raises(TypeError, nditer, a)
assert_equal(sys.getrefcount(obj), rc)
if HAS_REFCOUNT:
assert_equal(sys.getrefcount(obj), rc)

i = nditer(a, ['refs_ok'], ['readonly'])
vals = [x_[()] for x_ in i]
assert_equal(np.array(vals, dtype='O'), a)
vals, i, x = [None]*3
assert_equal(sys.getrefcount(obj), rc)
if HAS_REFCOUNT:
assert_equal(sys.getrefcount(obj), rc)

i = nditer(a.reshape(2, 2).T, ['refs_ok', 'buffered'],
['readonly'], order='C')
assert_(i.iterationneedsapi)
vals = [x_[()] for x_ in i]
assert_equal(np.array(vals, dtype='O'), a.reshape(2, 2).ravel(order='F'))
vals, i, x = [None]*3
assert_equal(sys.getrefcount(obj), rc)
if HAS_REFCOUNT:
assert_equal(sys.getrefcount(obj), rc)

i = nditer(a.reshape(2, 2).T, ['refs_ok', 'buffered'],
['readwrite'], order='C')
for x in i:
x[...] = None
vals, i, x = [None]*3
assert_equal(sys.getrefcount(obj), rc-1)
if HAS_REFCOUNT:
assert_(sys.getrefcount(obj) == rc-1)
assert_equal(a, np.array([None]*4, dtype='O'))

def test_iter_object_arrays_conversions():
Expand Down Expand Up @@ -1061,10 +1067,12 @@ def test_iter_object_arrays_conversions():
i = nditer(a, ['refs_ok', 'buffered'], ['readwrite'],
casting='unsafe', op_dtypes='O')
ob = i[0][()] 10670
rc = sys.getrefcount(ob)
if HAS_REFCOUNT:
rc = sys.getrefcount(ob)
for x in i:
x[...] += 1
assert_equal(sys.getrefcount(ob), rc-1)
if HAS_REFCOUNT:
assert_(sys.getrefcount(ob) == rc-1)
assert_equal(a, np.arange(6)+98172489)

def test_iter_common_dtype():
Expand Down Expand Up @@ -1704,7 +1712,8 @@ def test_iter_buffered_cast_structured_type():
a[0] = (0.5, 0.5, [[0.5, 0.5, 0.5], [0.5, 0.5, 0.5]], 0.5)
a[1] = (1.5, 1.5, [[1.5, 1.5, 1.5], [1.5, 1.5, 1.5]], 1.5)
a[2] = (2.5, 2.5, [[2.5, 2.5, 2.5], [2.5, 2.5, 2.5]], 2.5)
rc = sys.getrefcount(a[0])
if HAS_REFCOUNT:
rc = sys.getrefcount(a[0])
i = nditer(a, ['buffered', 'refs_ok'], ['readonly'],
casting='unsafe',
op_dtypes=sdt)
Expand All @@ -1719,7 +1728,8 @@ def test_iter_buffered_cast_structured_type():
assert_equal(vals[1]['d'], 1.5)
assert_equal(vals[0].dtype, np.dtype(sdt))
vals, i, x = [None]*3
assert_equal(sys.getrefcount(a[0]), rc)
if HAS_REFCOUNT:
assert_equal(sys.getrefcount(a[0]), rc)

# struct type -> simple (takes the first value)
sdt = [('a', 'f4'), ('b', 'i8'), ('d', 'O')]
Expand Down
3 changes: 2 additions & 1 deletion numpy/core/tests/test_numeric.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from numpy.testing import (
TestCase, run_module_suite, assert_, assert_equal, assert_raises,
assert_raises_regex, assert_array_equal, assert_almost_equal,
assert_array_almost_equal, dec
assert_array_almost_equal, dec, HAS_REFCOUNT
)


Expand Down Expand Up @@ -2011,6 +2011,7 @@ def test_full(self):
self.check_function(np.full, 0)
self.check_function(np.full, 1)

@dec.skipif(not HAS_REFCOUNT, "python has no sys.getrefcount")
def test_for_reference_leak(self):
# Make sure we have an object for reference
dim = 1
Expand Down
Loading
0