8000 [3.5] bpo-32072: Fix issues with binary plists. (GH-4455) (#4656) · python/cpython@43f014d · GitHub
[go: up one dir, main page]
Skip to content

Commit 43f014d

Browse files
serhiy-storchakalarryhastings
serhiy-storchaka
authored and
larryhastings
committed
[3.5] bpo-32072: Fix issues with binary plists. (GH-4455) (#4656)
* [3.5] bpo-32072: Fix issues with binary plists. (GH-4455) * Fixed saving bytearrays. * Identical objects will be saved only once. * Equal references will be load as identical objects. * Added support for saving and loading recursive data structures. (cherry picked from commit a897aee) * Fix implementation dependent assertion in test_plistlib. (#4813) It is failed with an advanced optimizer.
1 parent 891c91d commit 43f014d

File tree

3 files changed

+114
-37
lines changed

3 files changed

+114
-37
lines changed

‎Lib/plistlib.py

Lines changed: 51 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,8 @@ def __init__(self, message="Invalid file"):
590590

591591
_BINARY_FORMAT = {1: 'B', 2: 'H', 4: 'L', 8: 'Q'}
592592

593+
_undefined = object()
594+
593595
class _BinaryPlistParser:
594596
"""
595597
Read or write a binary plist file, following the description of the binary
@@ -620,7 +622,8 @@ def parse(self, fp):
620622
) = struct.unpack('>6xBBQQQ', trailer)
621623
self._fp.seek(offset_table_offset)
622624
self._object_offsets = self._read_ints(num_objects, offset_size)
623-
return self._read_object(self._object_offsets[top_object])
625+
self._objects = [_undefined] * num_objects
626+
return self._read_object(top_object)
624627

625628
except (OSError, IndexError, struct.error):
626629
raise InvalidFileException()
@@ -646,71 +649,78 @@ def _read_ints(self, n, size):
646649
def _read_refs(self, n):
647650
return self._read_ints(n, self._ref_size)
648651

649-
def _read_object(self, offset):
652+
def _read_object(self, ref):
650653
"""
651-
read the object at offset.
654+
read the object by reference.
652655
653656
May recursively read sub-objects (content of an array/dict/set)
654657
"""
658+
result = self._objects[ref]
659+
if result is not _undefined:
660+
return result
661+
662+
offset = self._object_offsets[ref]
655663
self._fp.seek(offset)
656664
token = self._fp.read(1)[0]
657665
tokenH, tokenL = token & 0xF0, token & 0x0F
658666

659667
if token == 0x00:
660-
return None
668+
result = None
661669

662670
elif token == 0x08:
663-
return False
671+
result = False
664672

665673
elif token == 0x09:
666-
return True
674+
result = True
667675

668676
# The referenced source code also mentions URL (0x0c, 0x0d) and
669677
# UUID (0x0e), but neither can be generated using the Cocoa libraries.
670678

671679
elif token == 0x0f:
672-
return b''
680+
result = b''
673681

674682
elif tokenH == 0x10: # int
675-
return int.from_bytes(self._fp.read(1 << tokenL),
676-
'big', signed=tokenL >= 3)
683+
result = int.from_bytes(self._fp.read(1 << tokenL),
684+
'big', signed=tokenL >= 3)
677685

678686
elif token == 0x22: # real
679-
return struct.unpack('>f', self._fp.read(4))[0]
687+
result = struct.unpack('>f', self._fp.read(4))[0]
680688

681689
elif token == 0x23: # real
682-
return struct.unpack('>d', self._fp.read(8))[0]
690+
result = struct.unpack('>d', self._fp.read(8))[0]
683691

684692
elif token == 0x33: # date
685693
f = struct.unpack('>d', self._fp.read(8))[0]
686694
# timestamp 0 of binary plists corresponds to 1/1/2001
687695
# (year of Mac OS X 10.0), instead of 1/1/1970.
688-
return datetime.datetime(2001, 1, 1) + datetime.timedelta(seconds=f)
696+
result = (datetime.datetime(2001, 1, 1) +
697+
datetime.timedelta(seconds=f))
689698

690699
elif tokenH == 0x40: # data
691700
s = self._get_size(tokenL)
692701
if self._use_builtin_types:
693-
return self._fp.read(s)
702+
result = self._fp.read(s)
694703
else:
695-
return Data(self._fp.read(s))
704+
result = Data(self._fp.read(s))
696705

697706
elif tokenH == 0x50: # ascii string
698707
s = self._get_size(tokenL)
699708
result = self._fp.read(s).decode('ascii')
700-
return result
709+
result = result
701710

702711
elif tokenH == 0x60: # unicode string
703712
s = self._get_size(tokenL)
704-
return self._fp.read(s * 2).decode('utf-16be')
713+
result = self._fp.read(s * 2).decode('utf-16be')
705714

706715
# tokenH == 0x80 is documented as 'UID' and appears to be used for
707716
# keyed-archiving, not in plists.
708717

709718
elif tokenH == 0xA0: # array
710719
s = self._get_size(tokenL)
711720
obj_refs = self._read_refs(s)
712-
return [self._read_object(self._object_offsets[x])
713-
for x in obj_refs]
721+
result = []
722+
self._objects[ref] = result
723+
result.extend(self._read_object(x) for x in obj_refs)
714724

715725
# tokenH == 0xB0 is documented as 'ordset', but is not actually
716726
# implemented in the Apple reference code.
@@ -723,12 +733,15 @@ def _read_object(self, offset):
723733
key_refs = self._read_refs(s)
724734
obj_refs = self._read_refs(s)
725735
result = self._dict_type()
736+
self._objects[ref] = result
726737
for k, o in zip(key_refs, obj_refs):
727-
result[self._read_object(self._object_offsets[k])
728-
] = self._read_object(self._object_offsets[o])
729-
return result
738+
result[self._read_object(k)] = self._read_object(o)
730739

731-
raise InvalidFileException()
740+
else:
741+
raise InvalidFileException()
742+
743+
self._objects[ref] = result
744+
return result
732745

733746
def _count_to_size(count):
734747
if count < 1 << 8:
@@ -743,6 +756,8 @@ def _count_to_size(count):
743756
else:
744757
return 8
745758

759+
_scalars = (str, int, float, datetime.datetime, bytes)
760+
746761
class _BinaryPlistWriter (object):
747762
def __init__(self, fp, sort_keys, skipkeys):
748763
self._fp = fp
@@ -798,24 +813,25 @@ def _flatten(self, value):
798813
# First check if the object is in the object table, not used for
799814
# containers to ensure that two subcontainers with the same contents
800815
# will be serialized as distinct values.
801-
if isinstance(value, (
802-
str, int, float, datetime.datetime, bytes, bytearray)):
816+
if isinstance(value, _scalars):
803817
if (type(value), value) in self._objtable:
804818
return
805819

806820
elif isinstance(value, Data):
807821
if (type(value.data), value.data) in self._objtable:
808822
return
809823

824+
elif id(value) in self._objidtable:
825+
return
826+
810827
# Add to objectreference map
811828
refnum = len(self._objlist)
812829
self._objlist.append(value)
813-
try:
814-
if isinstance(value, Data):
815-
self._objtable[(type(value.data), value.data)] = refnum
816-
else:
817-
self._objtable[(type(value), value)] = refnum
818-
except TypeError:
830+
if isinstance(value, _scalars):
831+
self._objtable[(type(value), value)] = refnum
832+
elif isinstance(value, Data):
833+
self._objtable[(type(value.data), value.data)] = refnum
834+
else:
819835
self._objidtable[id(value)] = refnum
820836

821837
# And finally recurse into containers
@@ -842,12 +858,11 @@ def _flatten(self, value):
842858
self._flatten(o)
843859

844860
def _getrefnum(self, value):
845-
try:
846-
if isinstance(value, Data):
847-
return self._objtable[(type(value.data), value.data)]
848-
else:
849-
return self._objtable[(type(value), value)]
850-
except TypeError:
861+
if isinstance(value, _scalars):
862+
return self._objtable[(type(value), value)]
863+
elif isinstance(value, Data):
864+
return self._objtable[(type(value.data), value.data)]
865+
else:
851866
return self._objidtable[id(value)]
852867

853868
def _write_size(self, token, size):

‎Lib/test/test_plistlib.py

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,17 @@ def test_int(self):
170170
self.assertRaises(OverflowError, plistlib.dumps,
171171
pl, fmt=fmt)
172172

173+
def test_bytearray(self):
174+
for pl in (b'<binary gunk>', b"<lots of binary gunk>\0\1\2\3" * 10):
175+
for fmt in ALL_FORMATS:
176+
with self.subTest(pl=pl, fmt=fmt):
177+
data = plistlib.dumps(bytearray(pl), fmt=fmt)
178+
pl2 = plistlib.loads(data)
179+
self.assertIsInstance(pl2, bytes)
180+
self.assertEqual(pl2, pl)
181+
data2 = plistlib.dumps(pl2, fmt=fmt)
182+
self.assertEqual(data, data2)
183+
173184
def test_bytes(self):
174185
pl = self._create()
175186
data = plistlib.dumps(pl)
@@ -311,7 +322,8 @@ def test_tuple_members(self):
311322
'second': [1, 2],
312323
'third': [3, 4],
313324
})
314-
self.assertIsNot(pl2['first'], pl2['second'])
325+
if fmt != plistlib.FMT_BINARY:
326+
self.assertIsNot(pl2['first'], pl2['second'])
315327

316328
def test_list_members(self):
317329
pl = {
@@ -423,6 +435,9 @@ def test_xml_encodings(self):
423435
pl2 = plistlib.loads(data)
424436
self.assertEqual(dict(pl), dict(pl2))
425437

438+
439+
class TestBinaryPlistlib(unittest.TestCase):
440+
426441
def test_nonstandard_refs_size(self):
427442
# Issue #21538: Refs and offsets are 24-bit integers
428443
data = (b'bplist00'
@@ -435,6 +450,47 @@ def test_nonstandard_refs_size(self):
435450
b'\x00\x00\x00\x00\x00\x00\x00\x13')
436451
self.assertEqual(plistlib.loads(data), {'a': 'b'})
437452

453+
def test_dump_duplicates(self):
454+
# Test effectiveness of saving duplicated objects
455+
for x in (None, False, True, 12345, 123.45, 'abcde', b'abcde',
456+
datetime.datetime(2004, 10, 26, 10, 33, 33),
457+
plistlib.Data(b'abcde'), bytearray(b'abcde'),
458+
[12, 345], (12, 345), {'12': 345}):
459+
with self.subTest(x=x):
460+
data = plistlib.dumps([x]*1000, fmt=plistlib.FMT_BINARY)
461+
self.assertLess(len(data), 1100, repr(data))
462+
463+
def test_identity(self):
464+
for x in (None, False, True, 12345, 123.45, 'abcde', b'abcde',
465+
datetime.datetime(2004, 10, 26, 10, 33, 33),
466+
plistlib.Data(b'abcde'), bytearray(b'abcde'),
467+
[12, 345], (12, 345), {'12': 345}):
468+
with self.subTest(x=x):
469+
data = plistlib.dumps([x]*2, fmt=plistlib.FMT_BINARY)
470+
a, b = plistlib.loads(data)
471+
if isinstance(x, tuple):
472+
x = list(x)
473+
self.assertEqual(a, x)
474+
self.assertEqual(b, x)
475+
self.assertIs(a, b)
476+
477+
def test_cycles(self):
478+
# recursive list
479+
a = []
480+
a.append(a)
481+
b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY))
482+
self.assertIs(b[0], b)
483+
# recursive tuple
484+
a = ([],)
485+
a[0].append(a)
486+
b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY))
487+
self.assertIs(b[0][0], b)
488+
# recursive dict
489+
a = {}
490+
a['x'] = a
491+
b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY))
492+
self.assertIs(b['x'], b)
493+
438494
def test_large_timestamp(self):
439495
# Issue #26709: 32-bit timestamp out of range
440496
for ts in -2**31-1, 2**31:

‎Misc/NEWS.d/next/Library/2017-11-18-21-13-52.bpo-32072.nwDV8L.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
Fixed issues with binary plists:
2+
3+
* Fixed saving bytearrays.
4+
* Identical objects will be saved only once.
5+
* Equal references will be load as identical objects.
6+
* Added support for saving and loading recursive data structures.

0 commit comments

Comments
 (0)
0