10000 Merge pull request #45 from msgpack/purepython · pyway/msgpack-python@2330e6c · GitHub
[go: up one dir, main page]
Skip to content

Commit 2330e6c

Browse files
methanemethane
committed
Merge pull request msgpack#45 from msgpack/purepython
fallback enhancements.
2 parents 266eaf8 + 1951b19 commit 2330e6c

File tree

4 files changed

+68
-23
lines changed

4 files changed

+68
-23
lines changed

‎benchmark/benchmark.py

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
from msgpack import fallback
2+
try:
3+
from msgpack import _unpacker, _packer
4+
has_ext = True
5+
except ImportError:
6+
has_ext = False
7+
import timeit
8+
9+
10+
def profile(name, func):
11+
times = timeit.repeat(func, number=1000, repeat=4)
12+
times = ', '.join(["%8f" % t for t in times])
13+
print("%-30s %40s" % (name, times))
14+
15+
16+
def simple(name, data):
17+
if has_ext:
18+
profile("packing %s (ext)" % name, lambda: _packer.packb(data))
19+
profile('packing %s (fallback)' % name, lambda: fallback.packb(data))
20+
21+
data = fallback.packb(data)
22+
if has_ext:
23+
profile('unpacking %s (ext)' % name, lambda: _unpacker.unpackb(data))
24+
profile('unpacking %s (fallback)' % name, lambda: fallback.unpackb(data))
25+
26+
def main():
27+
simple("integers", [7]*10000)
28+
simple("bytes", [b'x'*n for n in range(100)]*10)
29+
simple("lists", [[]]*10000)
30+
simple("dicts", [{}]*10000)
31+
32+
main()

‎msgpack/fallback.py

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -360,16 +360,19 @@ def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
360360
self._fb_unpack(EX_SKIP, write_bytes)
361361
self._fb_unpack(EX_SKIP, write_bytes)
362362
return
363-
ret = []
364-
for i in xrange(n):
365-
ret.append((self._fb_unpack(EX_CONSTRUCT, write_bytes),
366-
self._fb_unpack(EX_CONSTRUCT, write_bytes)))
367363
if self.object_pairs_hook is not None:
368-
ret = self.object_pairs_hook(ret)
364+
ret = self.object_pairs_hook(
365+
(self._fb_unpack(EX_CONSTRUCT, write_bytes),
366+
self._fb_unpack(EX_CONSTRUCT, write_bytes))
367+
for _ in xrange(n)
368+
)
369369
else:
370-
ret = dict(ret)
371-
if self.object_hook is not None:
372-
ret = self.object_hook(ret)
370+
ret = {}
371+
for _ in xrange(n):
372+
key = self._fb_unpack(EX_CONSTRUCT, write_bytes)
373+
ret[key] = self._fb_unpack(EX_CONSTRUCT, write_bytes)
374+
if self.object_hook is not None:
375+
ret = self.object_hook(ret)
373376
return ret
374377
if execute == EX_SKIP:
375378
return
@@ -421,7 +424,7 @@ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
421424
raise TypeError("default must be callable")
422425
self._default = default
423426

424-
def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT):
427+
def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance):
425428
if nest_limit < 0:
426429
raise PackValueError("recursion limit exceeded")
427430
if obj is None:
@@ -454,6 +457,10 @@ def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT):
454457
raise PackValueError("Integer value out of range")
455458
if isinstance(obj, (Unicode, bytes)):
456459
if isinstance(obj, Unicode):
460+
if self.encoding is None:
461+
raise TypeError(
462+
"Can't encode unicode string: "
463+
"no encoding is specified")
457464
obj = obj.encode(self.encoding, self.unicode_errors)
458465
n = len(obj)
459466
if n <= 0x1f:

‎setup.py

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,12 @@ def build_extension(self, ext):
4646
print("Install Cython >= 0.16 or install msgpack from PyPI.")
4747
print("Falling back to pure Python implementation.")
4848
return
49-
return build_ext.build_extension(self, ext)
49+
try:
50+
return build_ext.build_extension(self, ext)
51+
except Exception as e:
52+
print("WARNING: Failed to compile extensiom modules.")
53+
print("msgpack uses fallback pure python implementation.")
54+
print(e)
5055

5156

5257
exec(open('msgpack/_version.py').read())
@@ -75,18 +80,19 @@ def __init__(self, *args, **kwargs):
7580
macros = [('__LITTLE_ENDIAN__', '1')]
7681

7782
ext_modules = []
78-
ext_modules.append(Extension('msgpack._packer',
79-
sources=['msgpack/_packer.cpp'],
80-
libraries=libraries,
81-
include_dirs=['.'],
82-
define_macros=macros,
83-
))
84-
ext_modules.append(Extension('msgpack._unpacker',
85-
sources=['msgpack/_unpacker.cpp'],
86-
libraries=libraries,
87-
include_dirs=['.'],
88-
define_macros=macros,
89-
E200 ))
83+
if not hasattr(sys, 'pypy_version_info'):
84+
ext_modules.append(Extension('msgpack._packer',
85+
sources=['msgpack/_packer.cpp'],
86+
libraries=libraries,
87+
include_dirs=['.'],
88+
define_macros=macros,
89+
))
90+
ext_modules.append(Extension('msgpack._unpacker',
91+
sources=['msgpack/_unpacker.cpp'],
92+
libraries=libraries,
93+
include_dirs=['.'],
94+
define_macros=macros,
95+
))
9096
del libraries, macros
9197

9298

‎test/test_pack.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ def test_odict():
151151
od = odict(seq)
152152
assert unpackb(packb(od), use_list=1) == dict(seq)
153153
def pair_hook(seq):
154-
return seq
154+
return list(seq)
155155
assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
156156

157157

0 commit comments

Comments
 (0)
0