8000 Add bin type support. · zhurs/msgpack-python@da12e17 · GitHub
[go: up one dir, main page]
Skip to content

Commit da12e17

Browse files
methanemethane
committed
Add bin type support.
1 parent f45d7b4 commit da12e17

File tree

6 files changed

+81
-26
lines changed

6 files changed

+81
-26
lines changed

‎msgpack/_packer.pyx

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ cdef extern from "pack.h":
2626
int msgpack_pack_array(msgpack_packer* pk, size_t l)
2727
int msgpack_pack_map(msgpack_packer* pk, size_t l)
2828
int msgpack_pack_raw(msgpack_packer* pk, size_t l)
29+
int msgpack_pack_bin(msgpack_packer* pk, size_t l)
2930
int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
3031

3132
cdef int DEFAULT_RECURSE_LIMIT=511
@@ -56,6 +57,9 @@ cdef class Packer(object):
5657
:param bool autoreset:
5758
Reset buffer after each pack and return it's content as `bytes`. (default: True).
5859
If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
60+
:param bool use_bin_type:
61+
Use bin type introduced in msgpack spec 2.0 for bytes.
62+
It also enable str8 type for unicode.
5963
"""
6064
cdef msgpack_packer pk
6165
cdef object _default
@@ -64,6 +68,7 @@ cdef class Packer(object):
6468
cdef char *encoding
6569
cdef char *unicode_errors
6670
cdef bool use_float
71+
cdef bool use_bin_type
6772
cdef bint autoreset
6873

6974
def __cinit__(self):
@@ -74,11 +79,13 @@ cdef class Packer(object):
7479
self.pk.buf_size = buf_size
7580
self.pk.length = 0
7681

77-
def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1):
82+
def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
83+
use_single_float=False, bint autoreset=1, bint use_bin_type=0):
7884
"""
7985
"""
8086
self.use_float = use_single_float
8187
self.autoreset = autoreset
88+
self.pk.use_bin_type = use_bin_type
8289
if default is not None:
8390
if not PyCallable_Check(default):
8491
raise TypeError("default must be a callable.")
@@ -110,6 +117,7 @@ cdef class Packer(object):
110117
cdef char* rawval
111118
cdef int ret
112119
cdef dict d
120+
cdef size_t L
113121

114122
if nest_limit < 0:
115123
raise PackValueError("recursion limit exceeded.")
@@ -140,9 +148,10 @@ cdef class Packer(object):
140148
ret = msgpack_pack_double(&self.pk, dval)
141149
elif PyBytes_Check(o):
142150
rawval = o
143-
ret = msgpack_pack_raw(&self.pk, len(o))
151+
L = len(o)
152+
ret = msgpack_pack_bin(&self.pk, L)
144153
if ret == 0:
145-
ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
154+
ret = msgpack_pack_raw_body(&self.pk, rawval, L)
146155
elif PyUnicode_Check(o):
147156
if not self.encoding:
148157
raise TypeError("Can't encode unicode string: no encoding is specified")
@@ -247,21 +256,26 @@ cdef class Packer(object):
247256
return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
248257

249258

250-
def pack(object o, object stream, default=None, str encoding='utf-8', str unicode_errors='strict'):
259+
def pack(object o, object stream,
260+
default=None, str encoding='utf-8', str unicode_errors='strict',
261+
bint use_single_float=False, bint use_bin_type=False):
251262
"""
252263
pack an object `o` and write it to stream)
253264
254265
See :class:`Packer` for options.
255266
"""
256-
packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
267+
packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
268+
use_single_float=use_single_float, use_bin_type=use_bin_type)
257269
stream.write(packer.pack(o))
258270

259-
def packb(object o, default=None, encoding='utf-8', str unicode_errors='strict', bint use_single_float=False):
271+
def packb(object o,
272+
default=None, str encoding='utf-8', str unicode_errors='strict',
273+
bint use_single_float=False, bint use_bin_type=False):
260274
"""
261275
pack o and return packed bytes
262276
263277
See :class:`Packer` for options.
264278
"""
265279
packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
266-
use_single_float=use_single_float)
280+
use_single_float=use_single_float, use_bin_type=use_bin_type)
267281
return packer.pack(o)

‎msgpack/pack.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,11 @@ typedef struct msgpack_packer {
3434
char *buf;
3535
size_t length;
3636
size_t buf_size;
37+
bool use_bin_type;
3738
} msgpack_packer;
3839

3940
typedef struct Packer Packer;
4041

41-
static inline int msgpack_pack_short(msgpack_packer* pk, short d);
4242
static inline int msgpack_pack_int(msgpack_packer* pk, int d);
4343
static inline int msgpack_pack_long(msgpack_packer* pk, long d);
4444
static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
@@ -68,6 +68,7 @@ static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
6868
static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
6969

7070
static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
71+
static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l);
7172
static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
7273

7374
static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)

‎msgpack/pack_template.h

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -667,7 +667,10 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
667667
if(l < 32) {
668668
unsigned char d = 0xa0 | (uint8_t)l;
669669
msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
670-
} else if(l < 65536) {
670+
} else if (x->use_bin_type && l < 256) { // str8 is new format introduced with bin.
671+
unsigned char buf[2] = {0xd9, (uint8_t)l};
672+
msgpack_pack_append_buffer(x, buf, 2);
673+
} else if (l < 65536) {
671674
unsigned char buf[3];
672675
buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l);
673676
msgpack_pack_append_buffer(x, buf, 3);
@@ -678,6 +681,28 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
678681
}
679682
}
680683

684+
/*
685+
* bin
686+
*/
687+
static inline int msgpack_pack_bin(msgpack_packer *x, size_t l)
688+
{
689+
if (!x->use_bin_type) {
690+
return msgpack_pack_raw(x, l)
691+
}
692+
if (l < 256) {
693+
unsigned char buf[2] = {0xc4, (unsigned char)l};
694+
msgpack_pack_append_buffer(x, buf, 2);
695+
} else if (l < 65536) {
696+
unsigned char buf[3] = {0xc5};
697+
_msgpack_store16(&buf[1], (uint16_t)l);
698+
msgpack_pack_append_buffer(x, buf, 3);
699+
} else {
700+
unsigned char buf[5] = {0xc6};
701+
_msgpack_store32(&buf[1], (uint32_t)l);
702+
msgpack_pack_append_buffer(x, buf, 5);
703+
}
704+
}
705+
681706
static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l)
682707
{
683708
msgpack_pack_append_buffer(x, (const unsigned char*)b, l);

‎msgpack/unpack.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,4 +226,13 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
226226
return 0;
227227
}
228228

229+
static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
230+
{
231+
PyObject *py = PyBytes_FromStringAndSize(p, l);
232+
if (!py)
233+
return -1;
234+
*o = py;
235+
return 0;
236+
}
237+
229238
#include "unpack_template.h"

‎msgpack/unpack_define.h

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -34,16 +34,17 @@ extern "C" {
3434
#endif
3535

3636

37+
// CS is first byte & 0x1f
3738
typedef enum {
3839
CS_HEADER = 0x00, // nil
3940

4041
//CS_ = 0x01,
4142
//CS_ = 0x02, // false
4243
//CS_ = 0x03, // true
4344

44-
//CS_ = 0x04,
45-
//CS_ = 0x05,
46-
//CS_ = 0x06,
45+
CS_BIN_8 = 0x04,
46+
CS_BIN_16 = 0x05,
47+
CS_BIN_32 = 0x06,
4748
//CS_ = 0x07,
4849

4950
//CS_ = 0x08,
@@ -59,22 +60,16 @@ typedef enum {
5960
CS_INT_32 = 0x12,
6061
CS_INT_64 = 0x13,
6162

62-
//CS_ = 0x14,
63-
//CS_ = 0x15,
64-
//CS_BIG_INT_16 = 0x16,
65-
//CS_BIG_INT_32 = 0x17,
66-
//CS_BIG_FLOAT_16 = 0x18,
67-
//CS_BIG_FLOAT_32 = 0x19,
63+
CS_RAW_8 = 0x19,
6864
CS_RAW_16 = 0x1a,
6965
CS_RAW_32 = 0x1b,
7066
CS_ARRAY_16 = 0x1c,
7167
CS_ARRAY_32 = 0x1d,
7268
CS_MAP_16 = 0x1e,
7369
CS_MAP_32 = 0x1f,
7470

75-
//ACS_BIG_INT_VALUE,
76-
//ACS_BIG_FLOAT_VALUE,
7771
ACS_RAW_VALUE,
72+
ACS_BIN_VALUE,
7873
} msgpack_unpack_state;
7974

8075

‎msgpack/unpack_template.h

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -151,8 +151,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
151151
*/ \
152152
goto _header_again
153153

154-
#define NEXT_CS(p) \
155-
((unsigned int)*p & 0x1f)
154+
#define NEXT_CS(p) ((unsigned int)*p & 0x1f)
156155

157156
#ifdef USE_CASE_RANGE
158157
#define SWITCH_RANGE_BEGIN switch(*p) {
@@ -185,9 +184,6 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
185184
push_simple_value(_false);
186185
case 0xc3: // true
187186
push_simple_value(_true);
188-
//case 0xc4:
189-
//case 0xc5:
190-
//case 0xc6:
191187
//case 0xc7:
192188
//case 0xc8:
193189
//case 0xc9:
@@ -202,12 +198,15 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
202198
case 0xd2: // signed int 32
203199
case 0xd3: // signed int 64
204200
again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
201+
case 0xc4: // bin 8
202+
case 0xc5: // bin 16
203+
case 0xc6: // bin 32
205204
//case 0xd4:
206205
//case 0xd5:
207206
//case 0xd6: // big integer 16
208207
//case 0xd7: // big integer 32
209208
//case 0xd8: // big float 16
210-
//case 0xd9: // big float 32
209+
case 0xd9: // raw 8
211210
case 0xda: // raw 16
212211
case 0xdb: // raw 32
213212
case 0xdc: // array 16
@@ -290,6 +289,18 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
290289
// // FIXME
291290
// push_variable_value(_big_float, data, n, trail);
292291

292+
case CS_BIN_8:
293+
again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero);
294+
case CS_BIN_16:
295+
again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero);
296+
case CS_BIN_32:
297+
again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero);
298+
case ACS_BIN_VALUE:
299+
_bin_zero:
300+
push_variable_value(_bin, data, n, trail);
301+
302+
case CS_RAW_8:
303+
again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero);
293304
case CS_RAW_16:
294305
again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
295306
case CS_RAW_32:

0 commit comments

Comments
 (0)
0