8000 Add StackError and FormatError (#331) · loude/msgpack-python@44254dd · GitHub
[go: up one dir, main page]
Skip to content

Commit 44254dd

Browse files
methanemethane
authored
Add StackError and FormatError (msgpack#331)
1 parent 8b6ce53 commit 44254dd

File tree

7 files changed

+119
-47
lines changed

7 files changed

+119
-47
lines changed

‎ChangeLog.rst

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,25 @@ Release Date: TBD
55

66

77
Important changes
8-
------------------
8+
-----------------
99

10-
Extension modules are merged. There is ``msgpack._msgpack`` instead of
11-
``msgpack._packer`` and ``msgpack._unpacker``. (#314)
10+
* unpacker: Default size limits is smaller than before to avoid DoS attack.
11+
If you need to handle large data, you need to specify limits manually. (#319)
1212

13-
unpacker: Default size limits is smaller than before to avoid DoS attack.
14-
If you need to handle large data, you need to specify limits manually.
1513

14+
Other changes
15+
-------------
1616

17+
* Extension modules are merged. There is ``msgpack._msgpack`` instead of
18+
``msgpack._packer`` and ``msgpack._unpacker``. (#314)
1719

18-
Other changes
19-
--------------
20+
* Add ``Unpacker.getbuffer()`` method. (#320)
2021

21-
Add ``Unpacker.getbuffer()`` method.
22+
* unpacker: ``msgpack.StackError`` is raised when input data contains too
23+
nested data. (#331)
2224

25+
* unpacker: ``msgpack.FormatError`` is raised when input data is not valid
26+
msgpack format. (#331)
2327

2428

2529
0.5.6

‎Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ cython:
77
cython --cplus msgpack/_cmsgpack.pyx
88

99
.PHONY: test
10-
test:
10+
test: cython
11+
pip install -e .
1112
pytest -v test
1213
MSGPACK_PUREPYTHON=1 pytest -v test
1314

‎msgpack/_unpacker.pyx

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ from msgpack.exceptions import (
1616
BufferFull,
1717
OutOfData,
1818
ExtraData,
19+
FormatError,
20+
StackError,
1921
)
2022
from msgpack import ExtType
2123

@@ -149,7 +151,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
149151
"""
150152
Unpack packed_bytes to object. Returns an unpacked object.
151153
152-
Raises `ValueError` when `packed` contains extra bytes.
154+
Raises ``ExtraData`` when *packed* contains extra bytes.
155+
Raises ``ValueError`` when *packed* is incomplete.
156+
Raises ``FormatError`` when *packed* is not valid msgpack.
157+
Raises ``StackError`` when *packed* contains too nested.
158+
Other exceptions can be raised during unpacking.
153159
154160
See :class:`Unpacker` for options.
155161
"""
@@ -187,6 +193,12 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
187193
raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
188194
return obj
189195
unpack_clear(&ctx)
196+
if ret == 0:
197+
raise ValueError("Unpack failed: incomplete input")
198+
elif ret == -2:
199+
raise FormatError
200+
elif ret == -3:
201+
raise StackError
190202
raise ValueError("Unpack failed: error = %d" % (ret,))
191203

192204

@@ -201,7 +213,7 @@ def unpack(object stream, **kwargs):
201213
cdef class Unpacker(object):
202214
"""Streaming unpacker.
203215
204-
arguments:
216+
Arguments:
205217
206218
:param file_like:
207219
File-like object having `.read(n)` method.
@@ -279,6 +291,12 @@ cdef class Unpacker(object):
279291
unpacker.feed(buf)
280292
for o in unpacker:
281293
process(o)
294+
295+
Raises ``ExtraData`` when *packed* contains extra bytes.
296+
Raises ``OutOfData`` when *packed* is incomplete.
297+
Raises ``FormatError`` when *packed* is not valid msgpack.
298+
Raises ``StackError`` when *packed* contains too nested.
299+
Other exceptions can be raised during unpacking.
282300
"""
283301
cdef unpack_context ctx
284302
cdef char* buf
@@ -451,6 +469,10 @@ cdef class Unpacker(object):
451469
raise StopIteration("No more data to unpack.")
452470
else:
453471
raise OutOfData("No more data to unpack.")
472+
elif ret == -2:
473+
raise FormatError
474+
elif ret == -3:
475+
raise StackError
454476
else:
455477
raise ValueError("Unpack failed: error = %d" % (ret,))
456478

‎msgpack/exceptions.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ class UnpackException(Exception):
66
Exception instead.
77
"""
88

9+
910
class BufferFull(UnpackException):
1011
pass
1112

@@ -14,6 +15,14 @@ class OutOfData(UnpackException):
1415
pass
1516

1617

18+
class FormatError(ValueError, UnpackException):
19+
"""Invalid msgpack format"""
20+
21+
22+
class StackError(ValueError, UnpackException):
23+
"""Too nested"""
24+
25+
1726
# Deprecated. Use ValueError instead
1827
UnpackValueError = ValueError
1928

@@ -24,6 +33,7 @@ class ExtraData(UnpackValueError):
2433
This exception is raised while only one-shot (not streaming)
2534
unpack.
2635
"""
36+
2737
def __init__(self, unpacked, extra):
2838
self.unpacked = unpacked
2939
self.extra = extra
@@ -32,7 +42,7 @@ def __str__(self):
3242
return "unpack(b) received extra data."
3343

3444

35-
#Deprecated. Use Exception instead to catch all exception during packing.
45+
# Deprecated. Use Exception instead to catch all exception during packing.
3646
PackException = Exception
3747
PackValueError = ValueError
3848
PackOverflowError = OverflowError

‎msgpack/fallback.py

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,16 @@ def dict_iteritems(d):
1818
def dict_iteritems(d):
1919
return d.iteritems()
2020

21+
if sys.version_info < (3, 5):
22+
# Ugly hack...
23+
RecursionError = RuntimeError
24+
25+
def _is_recursionerror(e):
26+
return len(e.args) == 1 and isinstance(e.args[0], str) and \
27+
e.args[0].startswith('maximum recursion depth exceeded')
28+
else:
29+
def _is_recursionerror(e):
30+
return True
2131

2232
if hasattr(sys, 'pypy_version_info'):
2333
# cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own
@@ -52,7 +62,10 @@ def getvalue(self):
5262
from msgpack.exceptions import (
5363
BufferFull,
5464
OutOfData,
55-
ExtraData)
65+
ExtraData,
66+
FormatError,
67+
StackError,
68+
)
5669

5770
from msgpack import ExtType
5871

@@ -109,15 +122,24 @@ def unpackb(packed, **kwargs):
109122
"""
110123
Unpack an object from `packed`.
111124
112-
Raises `ExtraData` when `packed` contains extra bytes.
125+
Raises ``ExtraData`` when *packed* contains extra bytes.
126+
Raises ``ValueError`` when *packed* is incomplete.
127+
Raises ``FormatError`` when *packed* is not valid msgpack.
128+
Raises ``StackError`` when *packed* contains too nested.
129+
Other exceptions can be raised during unpacking.
130+
113131
See :class:`Unpacker` for options.
114132
"""
115133
unpacker = Unpacker(None, **kwargs)
116134
unpacker.feed(packed)
117135
try:
118136
ret = unpacker._unpack()
119137
except OutOfData:
120-
raise ValueError("Data is not enough.")
138+
raise ValueError("Unpack failed: incomplete input")
139+
except RecursionError as e:
140+
if _is_recursionerror(e):
141+
raise StackError
142+
raise
121143
if unpacker._got_extradata():
122144
raise ExtraData(ret, unpacker._get_extradata())
123145
return ret
@@ -211,6 +233,12 @@ class Unpacker(object):
211233
unpacker.feed(buf)
212234
for o in unpacker:
213235
process(o)
236+
237+
Raises ``ExtraData`` when *packed* contains extra bytes.
238+
Raises ``OutOfData`` when *packed* is incomplete.
239+
Raises ``FormatError`` when *packed* is not valid msgpack.
240+
Raises ``StackError`` when *packed* contains too nested.
241+
Other exceptions can be raised during unpacking.
214242
"""
215243

216244
def __init__(self, file_like=None, read_size=0, use_list=True, raw=True,
@@ -561,7 +589,7 @@ def _read_header(self, execute=EX_CONSTRUCT):
561589
raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
562590
typ = TYPE_MAP
563591
else:
564-
raise ValueError("Unknown header: 0x%x" % b)
592+
raise FormatError("Unknown header: 0x%x" % b)
565593
return typ, n, obj
566594

567595
def _unpack(self, execute=EX_CONSTRUCT):
@@ -637,6 +665,8 @@ def __next__(self):
637665
except OutOfData:
638666
self._consume()
639667
raise StopIteration
668+
except RecursionError:
669+
raise StackError
640670

641671
next = __next__
642672

@@ -645,7 +675,10 @@ def skip(self):
645675
self._consume()
646676

647677
def unpack(self):
648-
ret = self._unpack(EX_CONSTRUCT)
678+
try:
679+
ret = self._unpack(EX_CONSTRUCT)
680+
except RecursionError:
681+
raise StackError
649682
self._consume()
650683
return ret
651684

‎msgpack/unpack_template.h

Lines changed: 5 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
123123
goto _fixed_trail_again
124124

125125
#define start_container(func, count_, ct_) \
126-
if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
126+
if(top >= MSGPACK_EMBED_STACK_SIZE) { ret = -3; goto _end; } \
127127
if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
128128
if((count_) == 0< 10000 /span>) { obj = stack[top].obj; \
129129
if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \
@@ -132,27 +132,6 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
132132
stack[top].size = count_; \
133133
stack[top].count = 0; \
134134
++top; \
135-
/*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
136-
/*printf("stack push %d\n", top);*/ \
137-
/* FIXME \
138-
if(top >= stack_size) { \
139-
if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
140-
size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \
141-
size_t nsize = csize * 2; \
142-
unpack_stack* tmp = (unpack_stack*)malloc(nsize); \
143-
if(tmp == NULL) { goto _failed; } \
144-
memcpy(tmp, ctx->stack, csize); \
145-
ctx->stack = stack = tmp; \
146-
ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
147-
} else { \
148-
size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \
149-
unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \
150-
if(tmp == NULL) { goto _failed; } \
151-
ctx->stack = stack = tmp; \
152-
ctx->stack_size = stack_size = stack_size * 2; \
153-
} \
154-
} \
155-
*/ \
156135
goto _header_again
157136

158137
#define NEXT_CS(p) ((unsigned int)*p & 0x1f)
@@ -229,7 +208,8 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
229208
case 0xdf: // map 32
230209
again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
231210
default:
232-
goto _failed;
211+
ret = -2;
212+
goto _end;
233213
}
234214
SWITCH_RANGE(0xa0, 0xbf) // FixRaw
235215
again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
@@ -239,7 +219,8 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
239219
start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
240220

241221
SWITCH_RANGE_DEFAULT
242-
goto _failed;
222+
ret = -2;
223+
goto _end;
243224
SWITCH_RANGE_END
244225
// end CS_HEADER
245226

‎test/test_except.py

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# coding: utf-8
33

44
from pytest import raises
5-
from msgpack import packb, unpackb
5+
from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData
66

77
import datetime
88

@@ -19,13 +19,34 @@ def test_raise_on_find_unsupported_value():
1919
def test_raise_from_object_hook():
2020
def hook(obj):
2121
raise DummyException
22+
2223
raises(DummyException, unpackb, packb({}), object_hook=hook)
23-
raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_hook=hook)
24-
raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_pairs_hook=hook)
25-
raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_hook=hook)
26-
raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_pairs_hook=hook)
24+
raises(DummyException, unpackb, packb({"fizz": "buzz"}), object_hook=hook)
25+
raises(DummyException, unpackb, packb({"fizz": "buzz"}), object_pairs_hook=hook)
26+
raises(DummyException, unpackb, packb({"fizz": {"buzz": "spam"}}), object_hook=hook)
27+
raises(
28+
DummyException,
29+
unpackb,
30+
packb({"fizz": {"buzz": "spam"}}),
31+
object_pairs_hook=hook,
32+
)
2733

2834

2935
def test_invalidvalue():
36+
incomplete = b"\xd9\x97#DL_" # raw8 - length=0x97
3037
with raises(ValueError):
31-
unpackb(b'\xd9\x97#DL_')
38+
unpackb(incomplete)
39+
40+
with raises(OutOfData):
41+
unpacker = Unpacker()
42+
unpacker.feed(incomplete)
43+
unpacker.unpack()
44+
45+
with raises(FormatError):
46+
unpackb(b"\xc1") # (undefined tag)
47+
48+
with raises(FormatError):
49+
unpackb(b"\x91\xc1") # fixarray(len=1) [ (undefined tag) ]
50+
51+
with raises(StackError):
52+
unpackb(b"\x91" * 3000) # nested fixarray(len=1)

0 commit comments

Comments
 (0)
0