8000 Packing ExtType and some cleanup · zhurs/msgpack-python@96bcd76 · GitHub
[go: up one dir, main page]
Skip to content

Commit 96bcd76

Browse files
methanemethane
committed
Packing ExtType and some cleanup
1 parent aa68c9b commit 96bcd76

File tree

7 files changed

+132
-170
lines changed

7 files changed

+132
-170
lines changed

‎msgpack/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ def pack(o, stream, **kwargs):
2626
packer = Packer(**kwargs)
2727
stream.write(packer.pack(o))
2828

29+
2930
def packb(o, **kwargs):
3031
"""
3132
Pack object `o` and return packed bytes
@@ -40,4 +41,3 @@ def packb(o, **kwargs):
4041

4142
dump = pack
4243
dumps = packb
43-

‎msgpack/_packer.pyx

Lines changed: 78 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ from libc.limits cimport *
88
from libc.stdint cimport int8_t
99

1010
from msgpack.exceptions import PackValueError
11+
from msgpack import ExtType
12+
1113

1214
cdef extern from "pack.h":
1315
struct msgpack_packer:
@@ -120,80 +122,87 @@ cdef class Packer(object):
120122
cdef int ret
121123
cdef dict d
122124
cdef size_t L
125+
cdef int default_used = 0
123126

124127
if nest_limit < 0:
125128
raise PackValueError("recursion limit exceeded.")
126129

127-
if o is None:
128-
ret = msgpack_pack_nil(&self.pk)
129-
elif isinstance(o, bool):
130-
if o:
131-
ret = msgpack_pack_true(&self.pk)
132-
else:
133-
ret = msgpack_pack_false(&self.pk)
134-
elif PyLong_Check(o):
135-
if o > 0:
136-
ullval = o
137-
ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
138-
else:
139-
llval = o
140-
ret = msgpack_pack_long_long(&self.pk, llval)
141-
elif PyInt_Check(o):
142-
longval = o
143-
ret = msgpack_pack_long(&self.pk, longval)
144-
elif PyFloat_Check(o):
145-
if self.use_float:
146-
fval = o
147-
ret = msgpack_pack_float(&self.pk, fval)
148-
else:
149-
dval = o
150-
ret = msgpack_pack_double(&self.pk, dval)
151-
elif PyBytes_Check(o):
152-
rawval = o
153-
L = len(o)
154-
ret = msgpack_pack_bin(&self.pk, L)
155-
if ret == 0:
130+
while True:
131+
if o is None:
132+
ret = msgpack_pack_nil(&self.pk)
133+
elif isinstance(o, bool):
134+
if o:
135+
ret = msgpack_pack_true(&self.pk)
136+
else:
137+
ret = msgpack_pack_false(&self.pk)
138+
elif PyLong_Check(o):
139+
if o > 0:
140+
ullval = o
141+
ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
142+
else:
143+
llval = o
144+
ret = msgpack_pack_long_long(&self.pk, llval)
145+
elif PyInt_Check(o):
146+
longval = o
147+
ret = msgpack_pack_long(&self.pk, longval)
148+
elif PyFloat_Check(o):
149+
if self.use_float:
150+
fval = o
151+
ret = msgpack_pack_float(&self.pk, fval)
152+
else:
153+
dval = o
154+
ret = msgpack_pack_double(&self.pk, dval)
155+
elif PyBytes_Check(o):
156+
rawval = o
157+
L = len(o)
158+
ret = msgpack_pack_bin(&self.pk, L)
159+
if ret == 0:
160+
ret = msgpack_pack_raw_body(&self.pk, rawval, L)
161+
elif PyUnicode_Check(o):
162+
if not self.encoding:
163+
raise TypeError("Can't encode unicode string: no encoding is specified")
164+
o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
165+
rawval = o
166+
ret = msgpack_pack_raw(&self.pk, len(o))
167+
if ret == 0:
168+
ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
169+
elif PyDict_CheckExact(o):
170+
d = <dict>o
171+
ret = msgpack_pack_map(&self.pk, len(d))
172+
if ret == 0:
173+
for k, v in d.iteritems():
174+
ret = self._pack(k, nest_limit-1)
175+
if ret != 0: break
176+
ret = self._pack(v, nest_limit-1)
177+
if ret != 0: break
178+
elif PyDict_Check(o):
179+
ret = msgpack_pack_map(&self.pk, len(o))
180+
if ret == 0:
181+
for k, v in o.items():
182+
ret = self._pack(k, nest_limit-1)
183+
if ret != 0: break
184+
ret = self._pack(v, nest_limit-1)
185+
if ret != 0: break
186+
elif isinstance(o, ExtType):
187+
# This should be before Tuple because ExtType is namedtuple.
188+
longval = o[0]
189+
rawval = o[1]
190+
L = len(o[1])
191+
ret = msgpack_pack_ext(&self.pk, longval, L)
156192
ret = msgpack_pack_raw_body(&self.pk, rawval, L)
157-
elif PyUnicode_Check(o):
158-
if not self.encoding:
159-
raise TypeError("Can't encode unicode string: no encoding is specified")
160-
o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
161-
rawval = o
162-
ret = msgpack_pack_raw(&self.pk, len(o))
163-
if ret == 0:
164-
ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
165-
elif PyDict_CheckExact(o):
166-
d = <dict>o
167-
ret = msgpack_pack_map(&self.pk, len(d))
168-
if ret == 0:
169-
for k, v in d.iteritems():
170-
ret = self._pack(k, nest_limit-1)
171-
if ret != 0: break
172-
ret = self._pack(v, nest_limit-1)
173-
if ret != 0: break
174-
elif PyDict_Check(o):
175-
ret = msgpack_pack_map(&self.pk, len(o))
176-
if ret == 0:
177-
for k, v in o.items():
178-
ret = self._pack(k, nest_limit-1)
179-
if ret != 0: break
180-
ret = self._pack(v, nest_limit-1)
181-
if ret != 0: break
182-
elif PyTuple_Check(o) or PyList_Check(o):
183-
ret = msgpack_pack_array(&self.pk, len(o))
184-
if ret == 0:
185-
for v in o:
186-
ret = self._pack(v, nest_limit-1)
187-
if ret != 0: break
188-
elif self.handle_unknown_type(o):
189-
# it means that obj was succesfully packed, so we are done
190-
return 0
191-
elif self._default:
192-
o = self._default(o)
193-
ret = self._pack(o, nest_limit-1)
194-
else:
195-
raise TypeError("can't serialize %r" % (o,))
196-
return ret
193+
elif PyTuple_Check(o) or PyList_Check(o):
194+
ret = msgpack_pack_array(&self.pk, len(o))
195+
if ret == 0:
196+
for v in o:
197+
ret = self._pack(v, nest_limit-1)
198+
if ret != 0: break
199+
elif not default_used and self._default:
200+
o = self._default(o)
201+
default_used = 1
202+
continue
203+
else:
204+
raise TypeError("can't serialize %r" % (o,))
205+
return ret
197206

198207
cpdef pack(self, object obj):
199208
cdef int ret
@@ -207,9 +216,6 @@ cdef class Packer(object):
207216
self.pk.length = 0
208217
return buf
209218

210-
def handle_unknown_type(self, obj):
211-
return None
212-
213219
def pack_ext_type(self, typecode, data):
214220
msgpack_pack_ext(&self.pk, typecode, len(data))
215221
msgpack_pack_raw_body(&self.pk, data, len(data))

‎msgpack/_unpacker.pyx

Lines changed: 13 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ from msgpack.exceptions import (
1616
UnpackValueError,
1717
ExtraData,
1818
)
19+
from msgpack import ExtType
1920

2021

2122
cdef extern from "unpack.h":
@@ -24,7 +25,7 @@ cdef extern from "unpack.h":
2425
PyObject* object_hook
2526
bint has_pairs_hook # call object_hook with k-v pairs
2627
PyObject* list_hook
27-
PyObject* ext_type_hook
28+
PyObject* ext_hook
2829
char *encoding
2930
char *unicode_errors
3031

@@ -43,8 +44,8 @@ cdef extern from "unpack.h":
4344
object unpack_data(unpack_context* ctx)
4445

4546
cdef inline init_ctx(unpack_context *ctx,
46-
object object_hook, object object_pairs_hook, object list_hook,
47-
object ext_type_hook,
47+
object object_hook, object object_pairs_hook,
48+
object list_hook, object ext_hook,
4849
bint use_list, char* encoding, char* unicode_errors):
4950
unpack_init(ctx)
5051
ctx.user.use_list = use_list
@@ -71,10 +72,10 @@ cdef inline init_ctx(unpack_context *ctx,
7172
raise TypeError("list_hook must be a callable.")
7273
ctx.user.list_hook = <PyObject*>list_hook
7374

74-
if ext_type_hook is not None:
75-
if not PyCallable_Check(ext_type_hook):
76-
raise TypeError("ext_type_hook must be a callable.")
77-
ctx.user.ext_type_hook = <PyObject*>ext_type_hook
75+
if ext_hook is not None:
76+
if not PyCallable_Check(ext_hook):
77+
raise TypeError("ext_hook must be a callable.")
78+
ctx.user.ext_hook = <PyObject*>ext_hook
7879

7980
ctx.user.encoding = encoding
8081
ctx.user.unicode_errors = unicode_errors
@@ -84,8 +85,7 @@ def default_read_extended_type(typecode, data):
8485

8586
def unpackb(object packed, object object_hook=None, object list_hook=None,
8687
bint use_list=1, encoding=None, unicode_errors="strict",
87-
object_pairs_hook=None,
88-
):
88+
object_pairs_hook=None, ext_hook=ExtType):
8989
"""
9090
Unpack packed_bytes to object. Returns an unpacked object.
9191
@@ -114,8 +114,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
114114
unicode_errors = unicode_errors.encode('ascii')
115115
cerr = PyBytes_AsString(unicode_errors)
116116

117-
init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, default_read_extended_type,
118-
use_list, cenc, cerr)
117+
init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
118+
use_list, cenc, cerr)
119119
ret = unpack_construct(&ctx, buf, buf_len, &off)
120120
if ret == 1:
121121
obj = unpack_data(&ctx)
@@ -220,7 +220,7 @@ cdef class Unpacker(object):
220220
def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
221221
object object_hook=None, object object_pairs_hook=None, object list_hook=None,
222222
str encoding=None, str unicode_errors='strict', int max_buffer_size=0,
223-
):
223+
object ext_hook=ExtType):
224224
cdef char *cenc=NULL, *cerr=NULL
225225

226226
self.file_like = file_like
@@ -257,10 +257,8 @@ cdef class Unpacker(object):
257257
self.unicode_errors = unicode_errors
258258
cerr = PyBytes_AsString(self.unicode_errors)
259259

260-
ext_type_hook = self.read_extended_type
261-
Py_INCREF(ext_type_hook)
262260
init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
263-
ext_type_hook, use_list, cenc, cerr)
261+
ext_hook, use_list, cenc, cerr)
264262

265263
def feed(self, object next_bytes):
266264
"""Append `next_bytes` to internal buffer."""
@@ -370,24 +368,6 @@ cdef class Unpacker(object):
370368
"""
371369
return self._unpack(unpack_construct, write_bytes)
372370

373-
def unpack_one(self, object write_bytes=None):
374-
"""
375-
unpack one object
376-
377-
If write_bytes is not None, it will be called with parts of the raw
378-
message as it is unpacked.
379-
380-
Raises `UnpackValueError` if there are no more bytes to unpack.
381-
Raises ``ExtraData`` if there are still bytes left after the unpacking.
382-
"""
383-
try:
384-
result = self.unpack()
385-
except OutOfData:
386-
raise UnpackValueError("Data is not enough")
387-
if self.buf_head < self.buf_tail:
388-
raise ExtraData(result, self.buf[self.buf_head:])
389-
return result
390-
391371
def skip(self, object write_bytes=None):
392372
"""
393373
read and ignore one object, returning None
@@ -415,9 +395,6 @@ cdef class Unpacker(object):
415395
"""
416396
return self._unpack(read_map_header, write_bytes)
417397

418-
def read_extended_type(self, typecode, data):
419-
return default_read_extended_type(typecode, data)
420-
421398
def __iter__(self):
422399
return self
423400

‎msgpack/pack_template.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -687,7 +687,7 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
687687
static inline int msgpack_pack_bin(msgpack_packer *x, size_t l)
688688
{
689689
if (!x->use_bin_type) {
690-
return msgpack_pack_raw(x, l)
690+
return msgpack_pack_raw(x, l);
691691
}
692692
if (l < 256) {
693693
unsigned char buf[2] = {0xc4, (unsigned char)l};
@@ -711,7 +711,6 @@ static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t
711711
/*
712712
* Ext
713713
*/
714-
715714
static inline int msgpack_pack_ext(msgpack_packer* x, int8_t typecode, size_t l)
716715
{
717716
if (l == 1) {

‎msgpack/unpack.h

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ typedef struct unpack_user {
2424
PyObject *object_hook;
2525
bool has_pairs_hook;
2626
PyObject *list_hook;
27-
PyObject *ext_type_hook;
27+
PyObject *ext_hook;
2828
const char *encoding;
2929
const char *unicode_errors;
3030
} unpack_user;
@@ -241,12 +241,12 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch
241241
{
242242
PyObject *py;
243243
int8_t typecode = (int8_t)*pos++;
244-
if (!u->ext_type_hook) {
245-
PyErr_SetString(PyExc_AssertionError, "u->ext_type_hook cannot be NULL");
244+
if (!u->ext_hook) {
245+
PyErr_SetString(PyExc_AssertionError, "u->ext_hook cannot be NULL");
246246
return -1;
247247
}
248-
// lenght also includes the typecode, so the actual data is lenght-1
249-
py = PyEval_CallFunction(u->ext_type_hook, "(is#)", typecode, pos, lenght-1);
248+
// length also includes the typecode, so the actual data is lenght-1
249+
py = PyEval_CallFunction(u->ext_hook, "(is#)", typecode, pos, lenght-1);
250250
if (!py)
251251
return -1;
252252
*o = py;

0 commit comments

Comments
 (0)
0