10000 bpo-45125: Improves pickling docs and tests for `shared_memory` (GH-2… · python/cpython@fc3511f · GitHub
[go: up one dir, main page]
Skip to content

Commit fc3511f

Browse files
miss-islingtonsobolevn
miss-islington
and
sobolevn
authored
bpo-45125: Improves pickling docs and tests for shared_memory (GH-28294)
(cherry picked from commit 746d648) Co-authored-by: Nikita Sobolev <mail@sobolevn.me>
1 parent b994fee commit fc3511f

File tree

3 files changed

+133
-31
lines changed

3 files changed

+133
-31
lines changed

‎Doc/library/multiprocessing.shared_memory.rst

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,3 +342,30 @@ behind it:
342342
>>> c.shm.close()
343343
>>> c.shm.unlink()
344344

345+
The following examples demonstrates that ``ShareableList``
346+
(and underlying ``SharedMemory``) objects
347+
can be pickled and unpickled if needed.
348+
Note, that it will still be the same shared object.
349+
This happens, because the deserialized object has
350+
the same unique name and is just attached to an existing
351+
object with the same name (if the object is still alive):
352+
353+
>>> import pickle
354+
>>> from multiprocessing import shared_memory
355+
>>> sl = shared_memory.ShareableList(range(10))
356+
>>> list(sl)
357+
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
358+
359+
>>> deserialized_sl = pickle.loads(pickle.dumps(sl))
360+
>>> list(deserialized_sl)
361+
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
362+
363+
>>> sl[0] = -1
364+
>>> deserialized_sl[1] = -2
365+
>>> list(sl)
366+
[-1, -2, 2, 3, 4, 5, 6, 7, 8, 9]
367+
>>> list(deserialized_sl)
368+
[-1, -2, 2, 3, 4, 5, 6, 7, 8, 9]
369+
370+
>>> sl.shm.close()
371+
>>> sl.shm.unlink()

‎Lib/test/_test_multiprocessing.py

Lines changed: 104 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -3793,13 +3793,6 @@ def test_shared_memory_basics(self):
37933793
self.assertIn(sms.name, str(sms))
37943794
self.assertIn(str(sms.size), str(sms))
37953795

3796-
# Test pickling
3797-
sms.buf[0:6] = b'pickle'
3798-
pickled_sms = pickle.dumps(sms)
3799-
sms2 = pickle.loads(pickled_sms)
3800-
self.assertEqual(sms.name, sms2.name)
3801-
self.assertEqual(bytes(sms.buf[0:6]), bytes(sms2.buf[0:6]), b'pickle')
3802-
38033796
# Modify contents of shared memory segment through memoryview.
38043797
sms.buf[0] = 42
38053798
self.assertEqual(sms.buf[0], 42)
@@ -3898,6 +3891,29 @@ class OptionalAttachSharedMemory(shared_memory.SharedMemory):
38983891

38993892
sms.close()
39003893

3894+
def test_shared_memory_recreate(self):
3895+
# Test if shared memory segment is created properly,
3896+
# when _make_filename returns an existing shared memory segment name
3897+
with unittest.mock.patch(
3898+
'multiprocessing.shared_memory._make_filename') as mock_make_filename:
3899+
3900+
NAME_PREFIX = shared_memory._SHM_NAME_PREFIX
3901+
names = ['test01_fn', 'test02_fn']
3902+
# Prepend NAME_PREFIX which can be '/psm_' or 'wnsm_', necessary
3903+
# because some POSIX compliant systems require name to start with /
3904+
names = [NAME_PREFIX + name for name in names]
3905+
3906+
mock_make_filename.side_effect = names
3907+
shm1 = shared_memory.SharedMemory(create=True, size=1)
3908+
self.addCleanup(shm1.unlink)
3909+
self.assertEqual(shm1._name, names[0])
3910+
3911+
mock_make_filename.side_effect = names
3912+
shm2 = shared_memory.SharedMemory(create=True, size=1)
3913+
self.addCleanup(shm2.unlink)
3914+
self.assertEqual(shm2._name, names[1])
3915+
3916+
def test_invalid_shared_memory_cration(self):
39013917
# Test creating a shared memory segment with negative size
39023918
with self.assertRaises(ValueError):
39033919
sms_invalid = shared_memory.SharedMemory(create=True, size=-1)
@@ -3910,6 +3926,47 @@ class OptionalAttachSharedMemory(shared_memory.SharedMemory):
39103926
with self.assertRaises(ValueError):
39113927
sms_invalid = shared_memory.SharedMemory(create=True)
39123928

3929+
def test_shared_memory_pickle_unpickle(self):
3930+
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3931+
with self.subTest(proto=proto):
3932+
sms = shared_memory.SharedMemory(create=True, size=512)
3933+
self.addCleanup(sms.unlink)
3934+
sms.buf[0:6] = b'pickle'
3935+
3936+
# Test pickling
3937+
pickled_sms = pickle.dumps(sms, protocol=proto)
3938+
3939+
# Test unpickling
3940+
sms2 = pickle.loads(pickled_sms)
3941+
self.assertIsInstance(sms2, shared_memory.SharedMemory)
3942+
self.assertEqual(sms.name, sms2.name)
3943+
self.assertEqual(bytes(sms.buf[0:6]), b'pickle')
3944+
self.assertEqual(bytes(sms2.buf[0:6]), b'pickle')
3945+
3946+
# Test that unpickled version is still the same SharedMemory
3947+
sms.buf[0:6] = b'newval'
3948+
self.assertEqual(bytes(sms.buf[0:6]), b'newval')
3949+
self.assertEqual(bytes(sms2.buf[0:6]), b'newval')
3950+
3951+
sms2.buf[0:6] = b'oldval'
3952+
self.assertEqual(bytes(sms.buf[0:6]), b'oldval')
3953+
self.assertEqual(bytes(sms2.buf[0:6]), b'oldval')
3954+
3955+
def test_shared_memory_pickle_unpickle_dead_object(self):
3956+
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3957+
with self.subTest(proto=proto):
3958+
sms = shared_memory.SharedMemory(create=True, size=512)
3959+
sms.buf[0:6] = b'pickle'
3960+
pickled_sms = pickle.dumps(sms, protocol=proto)
3961+
3962+
# Now, we are going to kill the original object.
3963+
# So, unpickled one won't be able to attach to it.
3964+
sms.close()
3965+
sms.unlink()
3966+
3967+
with self.assertRaises(FileNotFoundError):
3968+
pickle.loads(pickled_sms)
3969+
39133970
def test_shared_memory_across_processes(self):
39143971
# bpo-40135: don't define shared memory block's name in case of
39153972
# the failure when we run multiprocessing tests in parallel.
@@ -4127,29 +4184,45 @@ def test_shared_memory_ShareableList_basics(self):
41274184
empty_sl.shm.unlink()
41284185

41294186
def test_shared_memory_ShareableList_pickling(self):
4130-
sl = shared_memory.ShareableList(range(10))
4131-
self.addCleanup(sl.shm.unlink)
4132-
4133-
serialized_sl = pickle.dumps(sl)
4134-
deserialized_sl = pickle.loads(serialized_sl)
4135-
self.assertTrue(
4136-
isinstance(deserialized_sl, shared_memory.ShareableList)
4137-
)
4138-
self.assertTrue(deserialized_sl[-1], 9)
4139-
self.assertFalse(sl is deserialized_sl)
4140-
deserialized_sl[4] = "changed"
4141-
self.assertEqual(sl[4], "changed")
4142-
4143-
# Verify data is not being put into the pickled representation.
4144-
name = 'a' * len(sl.shm.name)
4145-
larger_sl = shared_memory.ShareableList(range(400))
4146-
self.addCleanup(larger_sl.shm.unlink)
4147-
serialized_larger_sl = pickle.dumps(larger_sl)
4148-
self.assertTrue(len(serialized_sl) == len(serialized_larger_sl))
4149-
larger_sl.shm.close()
4150-
4151-
deserialized_sl.shm.close()
4152-
sl.shm.close()
4187+
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
4188+
with self.subTest(proto=proto):
4189+
sl = shared_memory.ShareableList(range(10))
4190+
self.addCleanup(sl.shm.unlink)
4191+
4192+
serialized_sl = pickle.dumps(sl, protocol=proto)
4193+
deserialized_sl = pickle.loads(serialized_sl)
4194+
self.assertIsInstance(
4195+
deserialized_sl, shared_memory.ShareableList)
4196+
self.assertEqual(deserialized_sl[-1], 9)
4197+
self.assertIsNot(sl, deserialized_sl)
4198+
4199+
deserialized_sl[4] = "changed"
4200+
self.assertEqual(sl[4], "changed")
4201+
sl[3] = "newvalue"
4202+
self.assertEqual(deserialized_sl[3], "newvalue")
4203+
4204+
larger_sl = shared_memory.ShareableList(range(400))
4205+
self.addCleanup(larger_sl.shm.unlink)
4206+
serialized_larger_sl = pickle.dumps(larger_sl, protocol=proto)
4207+
self.assertEqual(len(serialized_sl), len(serialized_larger_sl))
4208+
larger_sl.shm.close()
4209+
4210+
deserialized_sl.shm.close()
4211+
sl.shm.close()
4212+
4213+
def test_shared_memory_ShareableList_pickling_dead_object(self):
4214+
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
4215+
with self.subTest(proto=proto):
4216+
sl = shared_memory.ShareableList(range(10))
4217+
serialized_sl = pickle.dumps(sl, protocol=proto)
4218+
4219+
# Now, we are going to kill the original object.
4220+
# So, unpickled one won't be able to attach to it.
4221+
sl.shm.close()
4222+
sl.shm.unlink()
4223+
4224+
with self.assertRaises(FileNotFoundError):
4225+
pickle.loads(serialized_sl)
41534226

41544227
def test_shared_memory_cleaned_after_process_termination(self):
41554228
cmd = '''if 1:
@@ -4202,7 +4275,7 @@ def test_shared_memory_cleaned_after_process_termination(self):
42024275
"shared_memory objects to clean up at shutdown", err)
42034276

42044277
#
4205-
#
4278+
# Test to verify that `Finalize` works.
42064279
#
42074280

42084281
class _TestFinalize(BaseTestCase):

‎Misc/NEWS.d/next/Tests/2021-09-11-22-08-18.bpo-45125.FVSzs2.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Improves pickling tests and docs of ``SharedMemory`` and ``SharableList``
2+
objects.

0 commit comments

Comments
 (0)
0